Merge pull request #934 from biigle/metadata-improvements

Improve metadata import
biigle · Sep 30, 2024 · 6f4c117 · 6f4c117
2 parents db17160 + 7554d48
commit 6f4c117
Show file tree

Hide file tree

Showing 6 changed files with 79 additions and 8 deletions.
diff --git a/app/Jobs/ImportVolumeMetadata.php b/app/Jobs/ImportVolumeMetadata.php
@@ -126,7 +126,7 @@ protected function insertAnnotations(
 
             // Insert in chunks because a single file can have tens of thousands of
             // annotations (e.g. a video or mosaic).
-            if (($index % static::$insertChunkSize) === 0) {
+            if ($index > 0 && ($index % static::$insertChunkSize) === 0) {
                 $this->insertAnnotationChunk($file, $insertAnnotations, $insertAnnotationLabels);
                 $insertAnnotations = [];
                 $insertAnnotationLabels = [];
@@ -150,6 +150,7 @@ protected function insertAnnotationChunk(
             ->take(count($annotations))
             ->pluck('id')
             ->reverse()
+            ->values()
             ->toArray();
 
         foreach ($ids as $index => $id) {

diff --git a/app/Services/MetadataParsing/Annotation.php b/app/Services/MetadataParsing/Annotation.php
@@ -6,7 +6,8 @@
 use Biigle\Traits\HasPointsAttribute;
 use Exception;
 
-class Annotation
+// Abstract becaus it should not be used directly.
+abstract class Annotation
 {
     use HasPointsAttribute;
 
@@ -26,6 +27,13 @@ public function __construct(
         public array $labels,
     ) {
         $this->shape_id = $shape->id;
+        $this->setPointsAttribute($points);
+
+        array_walk($labels, function ($label) {
+            if (!($label instanceof LabelAndUser)) {
+                throw new Exception('Annotation labels must be of class '.LabelAndUser::class);
+            }
+        });
     }
 
     /**
@@ -60,6 +68,11 @@ public function validate(): void
      */
     public function setPointsAttribute(array $points)
     {
-        $this->points = array_map(fn ($coordinate) => round($coordinate, 2), $points);
+        $this->points = array_map(
+            fn ($a) => is_array($a)
+                ? array_map(fn ($b) => round($b, 2), $a)
+                : round($a, 2),
+            $points
+        );
     }
 }
diff --git a/app/Services/MetadataParsing/FileMetadata.php b/app/Services/MetadataParsing/FileMetadata.php
@@ -2,7 +2,8 @@
 
 namespace Biigle\Services\MetadataParsing;
 
-class FileMetadata
+// Abstract becaus it should not be used directly.
+abstract class FileMetadata
 {
     /**
      * @var array<Annotation>

diff --git a/app/Services/MetadataParsing/ImageMetadata.php b/app/Services/MetadataParsing/ImageMetadata.php
@@ -11,7 +11,8 @@ class ImageMetadata extends FileMetadata
      */
     public function isEmpty(): bool
     {
-        return is_null($this->lat)
+        return !$this->hasAnnotations()
+            && is_null($this->lat)
             && is_null($this->lng)
             && is_null($this->takenAt)
             && is_null($this->area)

diff --git a/tests/php/Jobs/ImportVolumeMetadataTest.php b/tests/php/Jobs/ImportVolumeMetadataTest.php
@@ -532,4 +532,60 @@ public function testHandleChunkAnnotations()
 
         $this->assertEquals(2, $image->annotations()->count());
     }
+
+    /*
+     * This tests if the annotations and their labels match after the import. The check is
+     * important because annotations and labels are inserted in two different DB
+     * statements and the ordering is important. The ordering was mixed up at some point
+     * so the annotations did not get the correct labels.
+     */
+    public function testHandleMultipleOrdering()
+    {
+        $image = Image::factory()->create();
+        $dbUser = DbUser::factory()->create();
+        $dbLabel = DbLabel::factory()->create();
+        $dbLabel2 = DbLabel::factory()->create();
+
+        $metadata = new VolumeMetadata;
+        $file = new ImageMetadata($image->filename);
+        $metadata->addFile($file);
+        $label = new Label(123, 'my label');
+        $user = new User(321, 'joe user');
+        $lau = new LabelAndUser($label, $user);
+        $file->addFileLabel($lau);
+        $annotation = new ImageAnnotation(
+            shape: Shape::point(),
+            points: [10, 10],
+            labels: [$lau],
+        );
+        $file->addAnnotation($annotation);
+
+        $label2 = new Label(321, 'my other label');
+        $lau2 = new LabelAndUser($label2, $user);
+        $annotation2 = new ImageAnnotation(
+            shape: Shape::point(),
+            points: [10, 10],
+            labels: [$lau2],
+        );
+        $file->addAnnotation($annotation2);
+
+        Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata);
+
+        $pv = PendingVolume::factory()->create([
+            'media_type_id' => MediaType::imageId(),
+            'metadata_file_path' => 'mymeta.csv',
+            'import_file_labels' => true,
+            'import_annotations' => true,
+            'user_map' => [321 => $dbUser->id],
+            'label_map' => [123 => $dbLabel->id, 321 => $dbLabel2->id],
+            'volume_id' => $image->volume_id,
+        ]);
+
+        (new ImportVolumeMetadata($pv))->handle();
+
+        $annotations = $image->annotations()->orderBy('id', 'asc')->get();
+
+        $this->assertSame($dbLabel->id, $annotations[0]->labels[0]->label_id);
+        $this->assertSame($dbLabel2->id, $annotations[1]->labels[0]->label_id);
+    }
 }
diff --git a/tests/php/Services/MetadataParsing/VolumeMetadataTest.php b/tests/php/Services/MetadataParsing/VolumeMetadataTest.php
@@ -4,7 +4,6 @@
 
 use Biigle\Label as DbLabel;
 use Biigle\MediaType;
-use Biigle\Services\MetadataParsing\FileMetadata;
 use Biigle\Services\MetadataParsing\ImageAnnotation;
 use Biigle\Services\MetadataParsing\ImageMetadata;
 use Biigle\Services\MetadataParsing\Label;
@@ -30,7 +29,7 @@ public function testNew()
     public function testAddGetFiles()
     {
         $metadata = new VolumeMetadata;
-        $file = new FileMetadata('filename');
+        $file = new ImageMetadata('filename');
         $metadata->addFile($file);
         $this->assertEquals($file, $metadata->getFiles()[0]);
         $metadata->addFile($file);
@@ -41,7 +40,7 @@ public function testGetFile()
     {
         $metadata = new VolumeMetadata;
         $this->assertNull($metadata->getFile('filename'));
-        $file = new FileMetadata('filename');
+        $file = new ImageMetadata('filename');
         $metadata->addFile($file);
         $this->assertEquals($file, $metadata->getFile('filename'));
     }