diff --git a/.docker/all-php.ini b/.docker/all-php.ini deleted file mode 100644 index ca8d0b189..000000000 --- a/.docker/all-php.ini +++ /dev/null @@ -1,3 +0,0 @@ -# Turn off serializing of PHP objects in YAML. -yaml.decode_php = 0 -yaml.decode_binary = 0 diff --git a/.docker/app.dockerfile b/.docker/app.dockerfile index ac825ecc4..c73d07a7c 100644 --- a/.docker/app.dockerfile +++ b/.docker/app.dockerfile @@ -5,7 +5,6 @@ LABEL org.opencontainers.image.authors="Martin Zurowietz "$PHP_INI_DIR/conf.d/vips.ini" @@ -58,19 +57,6 @@ RUN LC_ALL=C.UTF-8 apt-get update \ # Configure proxy if there is any. See: https://stackoverflow.com/a/2266500/1796523 RUN [ -z "$HTTP_PROXY" ] || pear config-set http_proxy $HTTP_PROXY -RUN LC_ALL=C.UTF-8 apt-get update \ - && apt-get install -y --no-install-recommends \ - libyaml-dev \ - && apt-get install -y --no-install-recommends \ - libyaml-0-2 \ - && pecl install yaml \ - && printf "\n" | docker-php-ext-enable yaml \ - && apt-get purge -y \ - libyaml-dev \ - && apt-get -y autoremove \ - && apt-get clean \ - && rm -r /var/lib/apt/lists/* - ARG PHPREDIS_VERSION=6.0.2 RUN curl -L -o /tmp/redis.tar.gz https://github.com/phpredis/phpredis/archive/${PHPREDIS_VERSION}.tar.gz \ && tar -xzf /tmp/redis.tar.gz \ diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 97b90c73c..f2a28ef7e 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -51,7 +51,7 @@ jobs: run: echo "APP_KEY=base64:STZFA4bQKDjE2mlpRPmsJ/okG0eCh4RHd9BghtZeYmQ=" >> .env - name: Run Linter - run: composer lint + run: composer lint -- --error-format=github cs-php: diff --git a/.gitignore b/.gitignore index de7c717d3..2cf34b81e 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,8 @@ /public/vendor /storage/*.key /storage/largo_patches +/storage/metadata +/storage/pending-metadata /vendor .env .env.backup diff --git a/app/Http/Controllers/Api/PendingVolumeController.php b/app/Http/Controllers/Api/PendingVolumeController.php new file mode 100644 index 000000000..c04a28809 --- /dev/null +++ b/app/Http/Controllers/Api/PendingVolumeController.php @@ -0,0 +1,203 @@ +project->pendingVolumes()->create([ + 'media_type_id' => $request->input('media_type_id'), + 'user_id' => $request->user()->id, + 'metadata_parser' => $request->input('metadata_parser', null), + ]); + + if ($request->has('metadata_file')) { + $pv->saveMetadata($request->file('metadata_file')); + } + + if ($this->isAutomatedRequest()) { + return $pv; + } + + return redirect()->route('pending-volume', $pv->id); + } + + /** + * Update a pending volume to create an actual volume + * + * @api {put} pending-volumes/:id Create a new volume (v2) + * @apiGroup Volumes + * @apiName UpdatePendingVolume + * @apiPermission projectAdminAndPendingVolumeOwner + * + * @apiDescription When this endpoint is called, the new volume is already created. Then there are two ways forward: 1) The user wants to import annotations and/or file labels. Then the pending volume is kept and used for the next steps (see the `import_*` attributes). Continue with (#Volumes:UpdatePendingVolumeAnnotationLabels) in this case. 2) Otherwise the pending volume will be deleted here. In both cases the endpoint returns the pending volume (even if it was deleted) which was updated with the new volume ID. + * + * @apiParam {Number} id The pending volume ID. + * + * @apiParam (Required attributes) {String} name The name of the new volume. + * @apiParam (Required attributes) {String} url The base URL of the image/video files. Can be a path to a storage disk like `local://volumes/1` or a remote path like `https://example.com/volumes/1`. + * @apiParam (Required attributes) {Array} files Array of file names of the images/videos that can be found at the base URL. Example: With the base URL `local://volumes/1` and the image `1.jpg`, the file `volumes/1/1.jpg` of the `local` storage disk will be used. This can also be a plain string of comma-separated filenames. + * + * @apiParam (Optional attributes) {String} handle Handle or DOI of the dataset that is represented by the new volume. + * @apiParam (Optional attributes) {Boolean} import_annotations Set to `true` to keep the pending volume for annotation import. Otherwise the pending volume will be deleted after this request. + * @apiParam (Optional attributes) {Boolean} import_file_labels Set to `true` to keep the pending volume for file label import. Otherwise the pending volume will be deleted after this request. + * + * @apiSuccessExample {json} Success response: + * { + * "id": 2, + * "created_at": "2015-02-19 16:10:17", + * "updated_at": "2015-02-19 16:10:17", + * "media_type_id": 1, + * "user_id": 2, + * "project_id": 3, + * "volume_id": 4, + * "import_annotations": true, + * "import_file_labels": false + * } + * + */ + public function update(UpdatePendingVolume $request) + { + $pv = $request->pendingVolume; + $volume = DB::transaction(function () use ($request, $pv) { + + $volume = Volume::create([ + 'name' => $request->input('name'), + 'url' => $request->input('url'), + 'media_type_id' => $pv->media_type_id, + 'handle' => $request->input('handle'), + 'creator_id' => $request->user()->id, + ]); + + $pv->project->volumes()->attach($volume); + + if ($pv->hasMetadata()) { + $volume->update([ + 'metadata_file_path' => $volume->id.'.'.pathinfo($pv->metadata_file_path, PATHINFO_EXTENSION), + 'metadata_parser' => $pv->metadata_parser, + ]); + $stream = Storage::disk(config('volumes.pending_metadata_storage_disk')) + ->readStream($pv->metadata_file_path); + Storage::disk(config('volumes.metadata_storage_disk')) + ->writeStream($volume->metadata_file_path, $stream); + } + + $files = $request->input('files'); + + // If too many files should be created, do this asynchronously in the + // background. Else the script will run in the 30s execution timeout. + $job = new CreateNewImagesOrVideos($volume, $files); + if (count($files) > self::CREATE_SYNC_LIMIT) { + Queue::pushOn('high', $job); + $volume->creating_async = true; + $volume->save(); + } else { + Queue::connection('sync')->push($job); + } + + return $volume; + }); + + if ($request->input('import_annotations') || $request->input('import_file_labels')) { + $pv->update([ + 'volume_id' => $volume->id, + 'import_annotations' => $request->input('import_annotations', false), + 'import_file_labels' => $request->input('import_file_labels', false), + ]); + } else { + $pv->volume_id = $volume->id; + $pv->delete(); + } + + if ($this->isAutomatedRequest()) { + return $pv; + } + + if ($pv->import_annotations) { + $redirect = redirect()->route('pending-volume-annotation-labels', $pv->id); + } elseif ($pv->import_file_labels) { + $redirect = redirect()->route('pending-volume-file-labels', $pv->id); + } else { + $redirect = redirect()->route('volume', $volume->id); + } + + return $redirect + ->with('message', 'Volume created.') + ->with('messageType', 'success'); + } + + /** + * Delete a pending volume + * + * @api {delete} pending-volumes/:id Discard a pending volume + * @apiGroup Volumes + * @apiName DestroyPendingVolume + * @apiPermission projectAdminAndPendingVolumeOwner + * + * @param Request $request] + */ + public function destroy(Request $request) + { + $pv = PendingVolume::findOrFail($request->route('id')); + $this->authorize('destroy', $pv); + + $pv->delete(); + + if (!$this->isAutomatedRequest()) { + return $this->fuzzyRedirect('create-volume', ['project' => $pv->project_id]); + } + } +} diff --git a/app/Http/Controllers/Api/PendingVolumeImportController.php b/app/Http/Controllers/Api/PendingVolumeImportController.php new file mode 100644 index 000000000..993e5a2d6 --- /dev/null +++ b/app/Http/Controllers/Api/PendingVolumeImportController.php @@ -0,0 +1,215 @@ +pendingVolume->update([ + 'only_annotation_labels' => $request->input('labels'), + ]); + + if ($this->isAutomatedRequest()) { + return $request->pendingVolume; + } + + if ($request->pendingVolume->import_file_labels) { + return redirect()->route('pending-volume-file-labels', $request->pendingVolume->id); + } + + return redirect()->route('pending-volume-label-map', $request->pendingVolume->id); + } + + /** + * Choose file labels for import. + * + * @api {put} pending-volumes/:id/file-labels Choose file labels for import + * @apiGroup Volumes + * @apiName UpdatePendingVolumeFileLabels + * @apiPermission projectAdminAndPendingVolumeOwner + * + * @apiDescription If this endpoint is not used to set a list of label IDs, all file labels will be imported by default. Continue with (#Volumes:UpdatePendingVolumeLabels). + * + * @apiParam {Number} id The pending volume ID. + * + * @apiParam (Required attributes) {array} labels The label IDs (from the metadata file) that should be used to filter the file label import. + * + * @apiSuccessExample {json} Success response: + * { + * "id": 2, + * "created_at": "2015-02-19 16:10:17", + * "updated_at": "2015-02-19 16:10:17", + * "media_type_id": 1, + * "user_id": 2, + * "project_id": 3, + * "volume_id": 4, + * "import_annotations": true, + * "import_file_labels": true, + * "only_annotation_labels": [123], + * "only_file_labels": [456] + * } + */ + public function updateFileLabels(UpdatePendingVolumeFileLabels $request) + { + $request->pendingVolume->update([ + 'only_file_labels' => $request->input('labels'), + ]); + + if ($this->isAutomatedRequest()) { + return $request->pendingVolume; + } + + return redirect()->route('pending-volume-label-map', $request->pendingVolume->id); + } + + /** + * Match metadata labels with database labels. + * + * @api {put} pending-volumes/:id/label-map Match metadata labels with database labels + * @apiGroup Volumes + * @apiName UpdatePendingVolumeLabels + * @apiPermission projectAdminAndPendingVolumeOwner + * + * @apiDescription If this endpoint is not used to set a map of metadata label IDs to database label IDs, the import will attempt to use the metadata label UUIDs to automatically find matches. Continue with (#Volumes:UpdatePendingVolumeUsers). + * + * @apiParam {Number} id The pending volume ID. + * + * @apiParam (Required attributes) {object} label_map Map of metadata label IDs as keys and database label IDs as values. + * + * @apiSuccessExample {json} Success response: + * { + * "id": 2, + * "created_at": "2015-02-19 16:10:17", + * "updated_at": "2015-02-19 16:10:17", + * "media_type_id": 1, + * "user_id": 2, + * "project_id": 3, + * "volume_id": 4, + * "import_annotations": true, + * "import_file_labels": true, + * "only_annotation_labels": [123], + * "only_file_labels": [456], + * "label_map": {"123": 987, "456": 654} + * } + */ + public function updateLabelMap(UpdatePendingVolumeLabelMap $request) + { + $map = array_map('intval', $request->input('label_map')); + $request->pendingVolume->update(['label_map' => $map]); + + if ($this->isAutomatedRequest()) { + return $request->pendingVolume; + } + + return redirect()->route('pending-volume-user-map', $request->pendingVolume->id); + } + + /** + * Match metadata users with database users. + * + * @api {put} pending-volumes/:id/user-map Match metadata users with database users + * @apiGroup Volumes + * @apiName UpdatePendingVolumeUsers + * @apiPermission projectAdminAndPendingVolumeOwner + * + * @apiDescription If this endpoint is not used to set a map of metadata user IDs to database user IDs, the import will attempt to use the metadata user UUIDs to automatically find matches. Continue with (#Volumes:UpdatePendingVolumeImport). + * + * @apiParam {Number} id The pending volume ID. + * + * @apiParam (Required attributes) {object} user_map Map of metadata user IDs as keys and database user IDs as values. + * + * @apiSuccessExample {json} Success response: + * { + * "id": 2, + * "created_at": "2015-02-19 16:10:17", + * "updated_at": "2015-02-19 16:10:17", + * "media_type_id": 1, + * "user_id": 2, + * "project_id": 3, + * "volume_id": 4, + * "import_annotations": true, + * "import_file_labels": true, + * "only_annotation_labels": [123], + * "only_file_labels": [456], + * "label_map": {"123": 987, "456": 654}, + * "user_map": {"135": 246, "975": 864} + * } + */ + public function updateUserMap(UpdatePendingVolumeUserMap $request) + { + $map = array_map('intval', $request->input('user_map')); + $request->pendingVolume->update(['user_map' => $map]); + + if ($this->isAutomatedRequest()) { + return $request->pendingVolume; + } + + return redirect()->route('pending-volume-finish', $request->pendingVolume->id); + } + + /** + * Perform the metadata annotation and/or file label import. + * + * @api {post} pending-volumes/:id/import Perform annotation/file label import + * @apiGroup Volumes + * @apiName UpdatePendingVolumeImport + * @apiPermission projectAdminAndPendingVolumeOwner + * + * @apiDescription This endpoint attempts to perform the annotation and/or file label import that can be started in (#Volumes:UpdatePendingVolume). If the import is successful, the pending volume will be deleted. + * + * @apiParam {Number} id The pending volume ID. + */ + public function storeImport(StorePendingVolumeImport $request) + { + DB::transaction(function () use ($request) { + $request->pendingVolume->update(['importing' => true]); + Queue::push(new ImportVolumeMetadata($request->pendingVolume)); + }); + + if (!$this->isAutomatedRequest()) { + return redirect() + ->route('volume', $request->pendingVolume->volume_id) + ->with('message', 'Metadata import in progress') + ->with('messageType', 'success'); + } + } +} diff --git a/app/Http/Controllers/Api/ProjectVolumeController.php b/app/Http/Controllers/Api/ProjectVolumeController.php index f5925cca3..6d683b961 100644 --- a/app/Http/Controllers/Api/ProjectVolumeController.php +++ b/app/Http/Controllers/Api/ProjectVolumeController.php @@ -12,13 +12,6 @@ class ProjectVolumeController extends Controller { - /** - * Limit for the number of files above which volume files are created asynchronously. - * - * @var int - */ - const CREATE_SYNC_LIMIT = 10000; - /** * Shows a list of all volumes belonging to the specified project.. * @@ -56,10 +49,11 @@ public function index($id) /** * Creates a new volume associated to the specified project. * - * @api {post} projects/:id/volumes Create a new volume + * @api {post} projects/:id/volumes Create a new volume (v1) * @apiGroup Volumes * @apiName StoreProjectVolumes * @apiPermission projectAdmin + * @apiDeprecated use now (#Volumes:StoreProjectPendingVolumes) and (#Volumes:UpdatePendingVolume). * * @apiParam {Number} id The project ID. * @@ -71,7 +65,6 @@ public function index($id) * @apiParam (Optional attributes) {String} handle Handle or DOI of the dataset that is represented by the new volume. * @apiParam (Optional attributes) {String} metadata_text CSV-like string with file metadata. See "metadata columns" for the possible columns. Each column may occur only once. There must be at least one column other than `filename`. For video metadata, multiple rows can contain metadata from different times of the same video. In this case, the `filename` of the rows must match and each row needs a (different) `taken_at` timestamp. * @apiParam (Optional attributes) {File} metadata_csv Alternative to `metadata_text`. This field allows the upload of an actual CSV file. See `metadata_text` for the further description. - * @apiParam (Optional attributes) {File} ifdo_file iFDO metadata file to upload and link with the volume. The metadata of this file is not used for the volume or volume files. Use `metadata_text` or `metadata_csv` for this. * * @apiParam (metadata columns) {String} filename The filename of the file the metadata belongs to. This column is required. * @apiParam (metadata columns) {String} taken_at The date and time where the file was taken. Example: `2016-12-19 12:49:00` @@ -113,18 +106,21 @@ public function store(StoreVolume $request) $volume->url = $request->input('url'); $volume->media_type_id = $request->input('media_type_id'); $volume->handle = $request->input('handle'); + $volume->metadata_parser = $request->metadataParser; $volume->creator()->associate($request->user()); $volume->save(); $request->project->volumes()->attach($volume); $files = $request->input('files'); - $metadata = $request->input('metadata', []); + if ($request->file('metadata_csv')) { + $volume->saveMetadata($request->file('metadata_csv')); + } // If too many files should be created, do this asynchronously in the // background. Else the script will run in the 30 s execution timeout. - $job = new CreateNewImagesOrVideos($volume, $files, $metadata); - if (count($files) > self::CREATE_SYNC_LIMIT) { + $job = new CreateNewImagesOrVideos($volume, $files); + if (count($files) > PendingVolumeController::CREATE_SYNC_LIMIT) { Queue::pushOn('high', $job); $volume->creating_async = true; $volume->save(); @@ -132,10 +128,6 @@ public function store(StoreVolume $request) Queue::connection('sync')->push($job); } - if ($request->hasFile('ifdo_file')) { - $volume->saveIfdo($request->file('ifdo_file')); - } - // media type shouldn't be returned unset($volume->media_type); diff --git a/app/Http/Controllers/Api/VolumeController.php b/app/Http/Controllers/Api/VolumeController.php index 41f03b3a0..9e80c3bed 100644 --- a/app/Http/Controllers/Api/VolumeController.php +++ b/app/Http/Controllers/Api/VolumeController.php @@ -192,6 +192,9 @@ public function clone(CloneVolume $request) $copy->name = $request->input('name', $volume->name); $copy->creating_async = true; $copy->save(); + if ($volume->hasMetadata()) { + $copy->update(['metadata_file_path' => $copy->id.'.'.pathinfo($volume->metadata_file_path, PATHINFO_EXTENSION)]); + } $project->addVolumeId($copy->id); $job = new CloneImagesOrVideos($request, $copy); diff --git a/app/Http/Controllers/Api/Volumes/IfdoController.php b/app/Http/Controllers/Api/Volumes/IfdoController.php deleted file mode 100644 index 11f3afac2..000000000 --- a/app/Http/Controllers/Api/Volumes/IfdoController.php +++ /dev/null @@ -1,46 +0,0 @@ -authorize('access', $volume); - - return $volume->downloadIfdo(); - } - - /** - * Delete an iFDO file attached to a volume - * - * @api {delete} volumes/:id/ifdo Delete an iFDO file - * @apiGroup Volumes - * @apiName DestroyVolumeIfdo - * @apiPermission projectAdmin - ~ - * @param int $id - */ - public function destroy($id) - { - $volume = Volume::findOrFail($id); - $this->authorize('update', $volume); - $volume->deleteIfdo(); - } -} diff --git a/app/Http/Controllers/Api/Volumes/MetadataController.php b/app/Http/Controllers/Api/Volumes/MetadataController.php index 4609a474a..5cab2cc8a 100644 --- a/app/Http/Controllers/Api/Volumes/MetadataController.php +++ b/app/Http/Controllers/Api/Volumes/MetadataController.php @@ -4,18 +4,40 @@ use Biigle\Http\Controllers\Api\Controller; use Biigle\Http\Requests\StoreVolumeMetadata; -use Biigle\Rules\ImageMetadata; -use Biigle\Rules\VideoMetadata; -use Biigle\Traits\ChecksMetadataStrings; -use Biigle\Video; -use Carbon\Carbon; -use DB; -use Illuminate\Support\Collection; -use Illuminate\Validation\ValidationException; +use Biigle\Jobs\UpdateVolumeMetadata; +use Biigle\Volume; +use Illuminate\Http\Response; +use Queue; +use Storage; class MetadataController extends Controller { - use ChecksMetadataStrings; + /** + * Get a metadata file attached to a volume + * + * @api {get} volumes/:id/metadata Get a metadata file + * @apiGroup Volumes + * @apiName ShowVolumeMetadata + * @apiPermission projectMember + ~ + * @param int $id + * + * @return \Symfony\Component\HttpFoundation\StreamedResponse + */ + public function show($id) + { + $volume = Volume::findOrFail($id); + $this->authorize('access', $volume); + + if (!$volume->hasMetadata()) { + abort(Response::HTTP_NOT_FOUND); + } + + $disk = Storage::disk(config('volumes.metadata_storage_disk')); + $suffix = pathinfo($volume->metadata_file_path, PATHINFO_EXTENSION); + + return $disk->download($volume->metadata_file_path, "biigle-volume-{$volume->id}-metadata.{$suffix}"); + } /** * @api {post} volumes/:id/images/metadata Add image metadata @@ -32,13 +54,14 @@ class MetadataController extends Controller * @apiGroup Volumes * @apiName StoreVolumeMetadata * @apiPermission projectAdmin - * @apiDescription This endpoint allows adding or updating metadata such as geo coordinates for volume file. + * @apiDescription This endpoint allows adding or updating metadata such as geo + * coordinates for volume files. The uploaded metadata file replaces any previously + * uploaded file. * * @apiParam {Number} id The volume ID. * - * @apiParam (Attributes) {String} metadata_text CSV-like string with file metadata. See "metadata columns" for the possible columns. Each column may occur only once. There must be at least one column other than `filename`. For video metadata, multiple rows can contain metadata from different times of the same video. In this case, the `filename` of the rows must match and each row needs a (different) `taken_at` timestamp. - * @apiParam (Attributes) {File} metadata_csv Alternative to `metadata_text`. This field allows the upload of an actual CSV file. See `metadata_text` for the further description. - * @apiParam (Attributes) {File} ifdo_file iFDO metadata file to upload and link with the volume. The metadata of this file is not used for the volume or volume files. Use `metadata_text` or `metadata_csv` for this. + * @apiParam (attributes) {File} file A file with volume and image/video metadata. By default, this can be a CSV. See "metadata columns" for the possible columns. Each column may occur only once. There must be at least one column other than `filename`. For video metadata, multiple rows can contain metadata from different times of the same video. In this case, the `filename` of the rows must match and each row needs a (different) `taken_at` timestamp. Other file formats may be supported through modules. + * @apiParam (attributes) {String} parser The class namespace of the metadata parser to use. The default CSV parsers are: `Biigle\Services\MetadataParsing\ImageCsvParser` and `Biigle\Services\MetadataParsing\VideoCsvParser`. * * @apiParam (metadata columns) {String} filename The filename of the file the metadata belongs to. This column is required. * @apiParam (metadata columns) {String} taken_at The date and time where the file was taken. Example: `2016-12-19 12:49:00` @@ -48,213 +71,36 @@ class MetadataController extends Controller * @apiParam (metadata columns) {Number} distance_to_ground Distance to the sea floor in meters. Example: `30.25` * @apiParam (metadata columns) {Number} area Area shown by the file in m². Example `2.6`. * - * @apiParamExample {String} Request example: - * file: "filename,taken_at,lng,lat,gps_altitude,distance_to_ground,area - * image_1.png,2016-12-19 12:49:00,52.3211,28.775,-1500.5,30.25,2.6" - * * @param StoreVolumeMetadata $request * * @return void */ public function store(StoreVolumeMetadata $request) { - if ($request->hasFile('ifdo_file')) { - $request->volume->saveIfdo($request->file('ifdo_file')); - } - - if ($request->input('metadata')) { - DB::transaction(function () use ($request) { - if ($request->volume->isImageVolume()) { - $this->updateImageMetadata($request); - } else { - $this->updateVideoMetadata($request); - } - }); - - $request->volume->flushGeoInfoCache(); - } - } - - /** - * Update volume metadata for each image. - * - * @param StoreVolumeMetadata $request - */ - protected function updateImageMetadata(StoreVolumeMetadata $request) - { - $metadata = $request->input('metadata'); - $images = $request->volume->images() - ->select('id', 'filename', 'attrs') - ->get() - ->keyBy('filename'); - - $columns = array_shift($metadata); - - foreach ($metadata as $row) { - $row = collect(array_combine($columns, $row)); - $image = $images->get($row['filename']); - // Remove empty cells. - $row = $row->filter(); - $fill = $row->only(ImageMetadata::ALLOWED_ATTRIBUTES); - if ($fill->has('taken_at')) { - $fill['taken_at'] = Carbon::parse($fill['taken_at']); - } - $image->fillable(ImageMetadata::ALLOWED_ATTRIBUTES); - $image->fill($fill->toArray()); - $metadata = $row - ->only(ImageMetadata::ALLOWED_METADATA) - ->map(fn ($str) => strpos($str, '.') ? floatval($str) : intval($str)); - $image->metadata = array_merge($image->metadata, $metadata->toArray()); - $image->save(); - } - } - - /** - * Update volume metadata for each video. - * - * @param StoreVolumeMetadata $request - */ - protected function updateVideoMetadata(StoreVolumeMetadata $request) - { - $metadata = $request->input('metadata'); - $videos = $request->volume->videos() - ->get() - ->keyBy('filename'); - - $columns = collect(array_shift($metadata)); - $rowsByFile = collect($metadata) - ->map(fn ($row) => $columns->combine($row)) - ->map(function ($row) { - if ($row->has('taken_at')) { - $row['taken_at'] = Carbon::parse($row['taken_at']); - } - - return $row; - }) - ->groupBy('filename'); - - foreach ($rowsByFile as $filename => $rows) { - $video = $videos->get($filename); - $merged = $this->mergeVideoMetadata($video, $rows); - $video->fillable(VideoMetadata::ALLOWED_ATTRIBUTES); - $video->fill($merged->only(VideoMetadata::ALLOWED_ATTRIBUTES)->toArray()); - // Fields for allowed metadata are filtered in mergeVideoMetadata(). We use - // except() with allowed attributes here so any metadata fields that were - // previously stored for the video but are not contained in ALLOWED_METADATA - // are not deleted. - $video->metadata = $merged->except(VideoMetadata::ALLOWED_ATTRIBUTES)->toArray(); - $video->save(); - } + // Delete first because the metadata file may have a different extension, so it + // is not guaranteed that the file is overwritten. + $request->volume->deleteMetadata(); + $request->volume->saveMetadata($request->file('file')); + $request->volume->update(['metadata_parser' => $request->input('metadata_parser')]); + Queue::push(new UpdateVolumeMetadata($request->volume)); } /** - * Merge existing video metadata with new metaddata based on timestamps. - * - * Timestamps of existing metadata are extended, even if no new values are provided - * for the fields. New values are extended with existing timestamps, even if these - * timestamps are not provided in the new metadata. + * Delete a metadata file attached to a volume * - * @param Video $video - * @param Collection $rows - * - * @return Collection - */ - protected function mergeVideoMetadata(Video $video, Collection $rows) - { - $metadata = collect(); - // Everything will be indexed by the timestamps below. - $origTakenAt = collect($video->taken_at)->map(fn ($time) => $time->getTimestamp()); - $newTakenAt = $rows->pluck('taken_at')->filter()->map(fn ($time) => $time->getTimestamp()); - - if ($origTakenAt->isEmpty() && $this->hasMetadata($video)) { - if ($rows->count() > 1 || $newTakenAt->isNotEmpty()) { - throw ValidationException::withMessages( - [ - 'metadata' => ["Metadata of video '{$video->filename}' has no 'taken_at' timestamps and cannot be updated with new metadata that has timestamps."], - ] - ); - } - - return $rows->first(); - } elseif ($newTakenAt->isEmpty()) { - throw ValidationException::withMessages( - [ - 'metadata' => ["Metadata of video '{$video->filename}' has 'taken_at' timestamps and cannot be updated with new metadata that has no timestamps."], - ] - ); - } - - // These are used to fill missing values with null. - $origTakenAtNull = $origTakenAt->combine($origTakenAt->map(fn ($x) => null)); - $newTakenAtNull = $newTakenAt->combine($newTakenAt->map(fn ($x) => null)); - - /** @var \Illuminate\Support\Collection */ - $originalAttributes = collect(VideoMetadata::ALLOWED_ATTRIBUTES) - ->mapWithKeys(fn ($key) => [$key => $video->$key]); - - /** @var \Illuminate\Support\Collection */ - $originalMetadata = collect(VideoMetadata::ALLOWED_METADATA) - ->mapWithKeys(fn ($key) => [$key => null]) - ->merge($video->metadata); - - $originalData = $originalMetadata->merge($originalAttributes); - - foreach ($originalData as $key => $originalValues) { - $originalValues = collect($originalValues); - if ($originalValues->isNotEmpty()) { - $originalValues = $origTakenAt->combine($originalValues); - } - - // Pluck returns an array filled with null if the key doesn't exist. - $newValues = $newTakenAt - ->combine($rows->pluck($key)) - ->filter([$this, 'isFilledString']); - - // This merges old an new values, leaving null where no values are given - // (for an existing or new timestamp). The union order is essential. - $newValues = $newValues - ->union($originalValues) - ->union($origTakenAtNull) - ->union($newTakenAtNull); - - // Do not insert completely empty new values. - if ($newValues->filter([$this, 'isFilledString'])->isEmpty()) { - continue; - } - - // Sort everything by ascending timestamps. - $metadata[$key] = $newValues->sortKeys()->values(); - } - - // Convert numeric fields to numbers. - foreach (VideoMetadata::NUMERIC_FIELDS as $key => $value) { - if ($metadata->has($key)) { - $metadata[$key]->transform(function ($x) { - // This check is required since floatval would return 0 for - // an empty value. This could skew metadata. - return $this->isFilledString($x) ? floatval($x) : null; - }); - } - } - - return $metadata; - } - - /** - * Determine if a video has any metadata. - * - * @param Video $video - * - * @return boolean + * @api {delete} volumes/:id/metadata Delete a metadata file + * @apiGroup Volumes + * @apiName DestroyVolumeMetadata + * @apiPermission projectAdmin + * @apiDescription This does not delete the metadata that was already attached to the + * volume files. + ~ + * @param int $id */ - protected function hasMetadata(Video $video) + public function destroy($id) { - foreach (VideoMetadata::ALLOWED_ATTRIBUTES as $key) { - if (!is_null($video->$key)) { - return true; - } - } - - return !empty($video->metadata); + $volume = Volume::findOrFail($id); + $this->authorize('update', $volume); + $volume->deleteMetadata(); } } diff --git a/app/Http/Controllers/Api/Volumes/ParseIfdoController.php b/app/Http/Controllers/Api/Volumes/ParseIfdoController.php deleted file mode 100644 index 530cd4b50..000000000 --- a/app/Http/Controllers/Api/Volumes/ParseIfdoController.php +++ /dev/null @@ -1,29 +0,0 @@ -metadata; - } -} diff --git a/app/Http/Controllers/Api/_apidoc.js b/app/Http/Controllers/Api/_apidoc.js index 686417f17..b5601b551 100644 --- a/app/Http/Controllers/Api/_apidoc.js +++ b/app/Http/Controllers/Api/_apidoc.js @@ -50,3 +50,8 @@ * The request must provide an authentication token of a remote instance configured for * federated search. */ + +/** + * @apiDefine projectAdminAndPendingVolumeOwner Project admin and pending volume owner + * The authenticated user must be admin of the project and creator of the pending volume. + */ diff --git a/app/Http/Controllers/Views/Volumes/PendingVolumeController.php b/app/Http/Controllers/Views/Volumes/PendingVolumeController.php new file mode 100644 index 000000000..c6dae116f --- /dev/null +++ b/app/Http/Controllers/Views/Volumes/PendingVolumeController.php @@ -0,0 +1,347 @@ +findOrFail($request->route('id')); + $this->authorize('access', $pv); + + // If the volume was already created, we have to redirect to one of the subsequent + // steps. + if (!is_null($pv->volume_id)) { + if ($pv->import_annotations && empty($pv->only_annotation_labels) && empty($pv->only_file_labels) && empty($pv->label_map) && empty($pv->user_map)) { + $redirect = redirect()->route('pending-volume-annotation-labels', $pv->id); + } elseif ($pv->import_file_labels && empty($pv->only_file_labels) && empty($pv->label_map) && empty($pv->user_map)) { + $redirect = redirect()->route('pending-volume-file-labels', $pv->id); + } elseif (empty($pv->label_map) && empty($pv->user_map)) { + $redirect = redirect()->route('pending-volume-label-map', $pv->id); + } else { + $redirect = redirect()->route('pending-volume-user-map', $pv->id); + } + + return $redirect + ->with('message', 'This is a pending volume that you did not finish before.') + ->with('messageType', 'info'); + } + + $disks = collect([]); + $user = $request->user(); + + if ($user->can('sudo')) { + $disks = $disks->concat(config('volumes.admin_storage_disks')); + } elseif ($user->role_id === Role::editorId()) { + $disks = $disks->concat(config('volumes.editor_storage_disks')); + } + + // Limit to disks that actually exist. + $disks = $disks->intersect(array_keys(config('filesystems.disks')))->values(); + + // Use the disk keys as names, too. UserDisks can have different names + // (see below). + $disks = $disks->combine($disks)->map(fn ($name) => ucfirst($name)); + + if (class_exists(UserDisk::class)) { + $userDisks = UserDisk::where('user_id', $user->id) + ->pluck('name', 'id') + ->mapWithKeys(fn ($name, $id) => ["disk-{$id}" => $name]); + + $disks = $disks->merge($userDisks); + } + + $offlineMode = config('biigle.offline_mode'); + + if (class_exists(UserStorageServiceProvider::class)) { + $userDisk = "user-{$user->id}"; + } else { + $userDisk = null; + } + + $isImageMediaType = $pv->media_type_id === MediaType::imageId(); + $mediaType = $isImageMediaType ? 'image' : 'video'; + + $metadata = null; + $oldName = ''; + $oldUrl = ''; + $oldHandle = ''; + if ($pv->hasMetadata()) { + $metadata = $pv->getMetadata(); + $oldName = $metadata->name; + $oldUrl = $metadata->url; + $oldHandle = $metadata->handle; + } + + $oldName = old('name', $oldName); + $oldUrl = old('url', $oldUrl); + $oldHandle = old('handle', $oldHandle); + + $filenamesFromMeta = false; + if ($filenames = old('files')) { + $filenames = str_replace(["\r", "\n", '"', "'"], '', old('files')); + } elseif ($metadata) { + $filenames = $metadata->getFiles()->pluck('name')->join(','); + $filenamesFromMeta = !empty($filenames); + } + + $hasAnnotations = $metadata && $metadata->hasAnnotations(); + $hasFileLabels = $metadata && $metadata->hasFileLabels(); + + return view('volumes.create.step2', [ + 'pv' => $pv, + 'project' => $pv->project, + 'disks' => $disks, + 'hasDisks' => $disks->isNotEmpty(), + 'filenames' => $filenames, + 'offlineMode' => $offlineMode, + 'userDisk' => $userDisk, + 'mediaType' => $mediaType, + 'isImageMediaType' => $isImageMediaType, + 'oldName' => $oldName, + 'oldUrl' => $oldUrl, + 'oldHandle' => $oldHandle, + 'filenamesFromMeta' => $filenamesFromMeta, + 'hasAnnotations' => $hasAnnotations, + 'hasFileLabels' => $hasFileLabels, + ]); + } + + /** + * Show the form to select labels of metadata annotations to import. + * + * @param Request $request + */ + public function showAnnotationLabels(Request $request) + { + $pv = PendingVolume::findOrFail($request->route('id')); + $this->authorize('update', $pv); + + if (is_null($pv->volume_id)) { + return redirect()->route('pending-volume', $pv->id); + } + + if (!$pv->hasMetadata()) { + abort(Response::HTTP_NOT_FOUND); + } + + $metadata = $pv->getMetadata(); + + if (!$metadata->hasAnnotations()) { + abort(Response::HTTP_NOT_FOUND); + } + + // Use values() for a more compact JSON representation. + $labels = collect($metadata->getAnnotationLabels())->values(); + + return view('volumes.create.annotationLabels', [ + 'pv' => $pv, + 'labels' => $labels, + ]); + } + + /** + * Show the form to select labels of metadata file labels to import. + * + * @param Request $request + */ + public function showFileLabels(Request $request) + { + $pv = PendingVolume::findOrFail($request->route('id')); + $this->authorize('update', $pv); + + if (is_null($pv->volume_id)) { + return redirect()->route('pending-volume', $pv->id); + } + + if (!$pv->hasMetadata()) { + abort(Response::HTTP_NOT_FOUND); + } + + $metadata = $pv->getMetadata(); + + if (!$metadata->hasFileLabels()) { + abort(Response::HTTP_NOT_FOUND); + } + + // Use values() for a more compact JSON representation. + $labels = collect($metadata->getFileLabels())->values(); + + return view('volumes.create.fileLabels', [ + 'pv' => $pv, + 'labels' => $labels, + ]); + } + + /** + * Show the form to select the label map for the metadata import. + * + * @param Request $request + */ + public function showLabelMap(Request $request) + { + $pv = PendingVolume::findOrFail($request->route('id')); + $this->authorize('update', $pv); + + if (is_null($pv->volume_id)) { + return redirect()->route('pending-volume', $pv->id); + } + + if (!$pv->hasMetadata()) { + abort(Response::HTTP_NOT_FOUND); + } + + $metadata = $pv->getMetadata(); + + $onlyLabels = $pv->only_annotation_labels + $pv->only_file_labels; + $labelMap = collect($metadata->getMatchingLabels(onlyLabels: $onlyLabels)); + + if ($labelMap->isEmpty()) { + abort(Response::HTTP_NOT_FOUND); + } + + // Merge with previously selected map on error. + $oldMap = collect(old('label_map', []))->map(fn ($v) => intval($v)); + $labelMap = $oldMap->union($labelMap); + + $labels = []; + + if ($pv->import_file_labels) { + $labels += $metadata->getFileLabels($pv->only_file_labels); + } + + if ($pv->import_annotations) { + $labels += $metadata->getAnnotationLabels($pv->only_annotation_labels); + } + + $labels = collect($labels)->values(); + + $project = $pv->project; + + // These label trees are required to display the pre-mapped labels. + $labelTrees = + LabelTree::whereIn( + 'id', + fn ($query) => + $query->select('label_tree_id') + ->from('labels') + ->whereIn('id', $labelMap->values()->unique()->filter()) + )->get()->keyBy('id'); + + // These trees can also be used for manual mapping. + $labelTrees = $labelTrees->union($project->labelTrees->keyBy('id'))->values(); + + $labelTrees->load('labels'); + + // Hide attributes for a more compact JSON representation. + $labelTrees->each(function ($tree) { + $tree->makeHidden(['visibility_id', 'created_at', 'updated_at']); + $tree->labels->each(function ($label) { + $label->makeHidden(['source_id', 'label_source_id', 'label_tree_id', 'parent_id']); + }); + }); + + return view('volumes.create.labelMap', [ + 'pv' => $pv, + 'labelMap' => $labelMap, + 'labels' => $labels, + 'labelTrees' => $labelTrees, + ]); + } + + /** + * Show the form to select the user map for the metadata import. + * + * @param Request $request + */ + public function showUserMap(Request $request) + { + $pv = PendingVolume::findOrFail($request->route('id')); + $this->authorize('update', $pv); + + if (is_null($pv->volume_id)) { + return redirect()->route('pending-volume', $pv->id); + } + + if (!$pv->hasMetadata()) { + abort(Response::HTTP_NOT_FOUND); + } + + $metadata = $pv->getMetadata(); + + $onlyLabels = $pv->only_annotation_labels + $pv->only_file_labels; + $userMap = collect($metadata->getMatchingUsers(onlyLabels: $onlyLabels)); + + if ($userMap->isEmpty()) { + abort(Response::HTTP_NOT_FOUND); + } + + // Merge with previously selected map on error. + $oldMap = collect(old('user_map', []))->map(fn ($v) => intval($v)); + $userMap = $oldMap->union($userMap); + + $users = collect($metadata->getUsers($onlyLabels)) + ->values() + ->pluck('name', 'id'); + + return view('volumes.create.userMap', [ + 'pv' => $pv, + 'userMap' => $userMap, + 'users' => $users, + ]); + } + + /** + * Show the view to finish the metadata import. + * + * @param Request $request + */ + public function showFinish(Request $request) + { + $pv = PendingVolume::findOrFail($request->route('id')); + $this->authorize('update', $pv); + + if (is_null($pv->volume_id)) { + return redirect()->route('pending-volume', $pv->id); + } + + if (!$pv->hasMetadata()) { + abort(Response::HTTP_NOT_FOUND); + } + + $metadata = $pv->getMetadata(); + + if (empty($metadata->getUsers())) { + abort(Response::HTTP_NOT_FOUND); + } + + $onlyLabels = $pv->only_annotation_labels + $pv->only_file_labels; + + $labelMap = $metadata->getMatchingLabels($pv->label_map, $onlyLabels); + $labelMapOk = !empty($labelMap) && array_search(null, $labelMap) === false; + + $userMap = $metadata->getMatchingUsers($pv->user_map, $onlyLabels); + $userMapOk = !empty($userMap) && array_search(null, $userMap) === false; + + return view('volumes.create.finish', [ + 'pv' => $pv, + 'labelMapOk' => $labelMapOk, + 'userMapOk' => $userMapOk, + ]); + } +} diff --git a/app/Http/Controllers/Views/Volumes/VolumeController.php b/app/Http/Controllers/Views/Volumes/VolumeController.php index 77c699398..d75651e19 100644 --- a/app/Http/Controllers/Views/Volumes/VolumeController.php +++ b/app/Http/Controllers/Views/Volumes/VolumeController.php @@ -5,10 +5,8 @@ use Biigle\Http\Controllers\Views\Controller; use Biigle\LabelTree; use Biigle\MediaType; -use Biigle\Modules\UserDisks\UserDisk; -use Biigle\Modules\UserStorage\UserStorageServiceProvider; use Biigle\Project; -use Biigle\Role; +use Biigle\Services\MetadataParsing\ParserFactory; use Biigle\User; use Biigle\Volume; use Carbon\Carbon; @@ -26,48 +24,31 @@ public function create(Request $request) $project = Project::findOrFail($request->input('project')); $this->authorize('update', $project); - $disks = collect([]); - $user = $request->user(); - - if ($user->can('sudo')) { - $disks = $disks->concat(config('volumes.admin_storage_disks')); - } elseif ($user->role_id === Role::editorId()) { - $disks = $disks->concat(config('volumes.editor_storage_disks')); - } - - // Limit to disks that actually exist. - $disks = $disks->intersect(array_keys(config('filesystems.disks')))->values(); - - // Use the disk keys as names, too. UserDisks can have different names - // (see below). - $disks = $disks->combine($disks)->map(fn ($name) => ucfirst($name)); - - if (class_exists(UserDisk::class)) { - $userDisks = UserDisk::where('user_id', $user->id) - ->pluck('name', 'id') - ->mapWithKeys(fn ($name, $id) => ["disk-{$id}" => $name]); - - $disks = $disks->merge($userDisks); + $pv = $project->pendingVolumes()->where('user_id', $request->user()->id)->first(); + if (!is_null($pv)) { + return redirect() + ->route('pending-volume', $pv->id) + ->with('message', 'This is a pending volume that you did not finish before.') + ->with('messageType', 'info'); } $mediaType = old('media_type', 'image'); - $filenames = str_replace(["\r", "\n", '"', "'"], '', old('files')); - $offlineMode = config('biigle.offline_mode'); - if (class_exists(UserStorageServiceProvider::class)) { - $userDisk = "user-{$user->id}"; - } else { - $userDisk = null; + $parsers = collect(ParserFactory::$parsers); + foreach ($parsers as $type => $p) { + $parsers[$type] = array_map(function ($class) { + return [ + 'parserClass' => $class, + 'name' => $class::getName(), + 'mimeTypes' => $class::getKnownMimeTypes(), + ]; + }, $p); } - return view('volumes.create', [ + return view('volumes.create.step1', [ 'project' => $project, - 'disks' => $disks, - 'hasDisks' => $disks->isNotEmpty(), 'mediaType' => $mediaType, - 'filenames' => $filenames, - 'offlineMode' => $offlineMode, - 'userDisk' => $userDisk, + 'parsers' => $parsers, ]); } @@ -128,6 +109,15 @@ public function edit(Request $request, $id) $projects = $this->getProjects($request->user(), $volume); $type = $volume->mediaType->name; + $parsers = collect(ParserFactory::$parsers[$type] ?? []) + ->map(function ($class) { + return [ + 'parserClass' => $class, + 'name' => $class::getName(), + 'mimeTypes' => $class::getKnownMimeTypes(), + ]; + }); + return view('volumes.edit', [ 'projects' => $projects, 'volume' => $volume, @@ -135,6 +125,7 @@ public function edit(Request $request, $id) 'annotationSessions' => $sessions, 'today' => Carbon::today(), 'type' => $type, + 'parsers' => $parsers, ]); } diff --git a/app/Http/Requests/StoreParseIfdo.php b/app/Http/Requests/StoreParseIfdo.php deleted file mode 100644 index faea4af2e..000000000 --- a/app/Http/Requests/StoreParseIfdo.php +++ /dev/null @@ -1,61 +0,0 @@ - 'required|file|max:500000', - ]; - } - - /** - * Configure the validator instance. - * - * @param \Illuminate\Validation\Validator $validator - * @return void - */ - public function withValidator($validator) - { - $validator->after(function ($validator) { - if ($this->hasFile('file')) { - try { - $this->metadata = $this->parseIfdoFile($this->file('file')); - } catch (Exception $e) { - $validator->errors()->add('file', $e->getMessage()); - } - } - }); - } -} diff --git a/app/Http/Requests/StorePendingVolume.php b/app/Http/Requests/StorePendingVolume.php new file mode 100644 index 000000000..ee9ffda41 --- /dev/null +++ b/app/Http/Requests/StorePendingVolume.php @@ -0,0 +1,115 @@ +project = Project::findOrFail($this->route('id')); + + return $this->user()->can('update', $this->project); + } + + /** + * Get the validation rules that apply to the request. + * + * @return array|string> + */ + public function rules(): array + { + + $rules = [ + 'media_type' => ['required', Rule::in(array_keys(MediaType::INSTANCES))], + 'metadata_parser' => [ + 'required_with:metadata_file', + ], + // Allow a maximum of 500 MB. + 'metadata_file' => [ + 'required_with:metadata_parser', + 'file', + 'max:500000', + ], + ]; + + $parserClass = $this->input('metadata_parser', false); + if ($this->has('media_type') && $parserClass && ParserFactory::has($this->input('media_type'), $parserClass)) { + $rules['metadata_file'][] = 'mimetypes:'.implode(',', $parserClass::getKnownMimeTypes()); + } + + return $rules; + } + + /** + * Configure the validator instance. + * + * @param \Illuminate\Validation\Validator $validator + * @return void + */ + public function withValidator($validator) + { + $validator->after(function ($validator) { + if ($validator->errors()->isNotEmpty()) { + return; + } + + $exists = $this->project->pendingVolumes() + ->where('user_id', $this->user()->id) + ->exists(); + if ($exists) { + $validator->errors()->add('id', 'Only a single pending volume can be created at a time for each project and user.'); + return; + } + + if ($file = $this->file('metadata_file')) { + $type = $this->input('media_type'); + $parserClass = $this->input('metadata_parser'); + + if (!ParserFactory::has($type, $parserClass)) { + $validator->errors()->add('metadata_parser', 'Unknown metadata parser for this media type.'); + return; + } + + $parser = new $parserClass($file); + if (!$parser->recognizesFile()) { + $validator->errors()->add('metadata_file', 'Unknown metadata file format.'); + return; + } + + $rule = match ($type) { + 'video' => new VideoMetadata, + default => new ImageMetadata, + }; + + if (!$rule->passes('metadata_file', $parser->getMetadata())) { + $validator->errors()->add('metadata_file', $rule->message()); + } + } + }); + } + + /** + * Prepare the data for validation. + * + * @return void + */ + protected function prepareForValidation() + { + // Allow a string as media_type to be more conventient. + $type = $this->input('media_type'); + if (in_array($type, array_keys(MediaType::INSTANCES))) { + $this->merge(['media_type_id' => MediaType::$type()->id]); + } + } +} diff --git a/app/Http/Requests/StorePendingVolumeImport.php b/app/Http/Requests/StorePendingVolumeImport.php new file mode 100644 index 000000000..ca9410e53 --- /dev/null +++ b/app/Http/Requests/StorePendingVolumeImport.php @@ -0,0 +1,142 @@ +pendingVolume = PendingVolume::findOrFail($this->route('id')); + + return $this->user()->can('update', $this->pendingVolume); + } + + /** + * Get the validation rules that apply to the request. + * + * @return array|string> + */ + public function rules(): array + { + return [ + // + ]; + } + + /** + * Configure the validator instance. + * + * @param \Illuminate\Validation\Validator $validator + * @return void + */ + public function withValidator($validator) + { + $validator->after(function ($validator) { + if ($validator->errors()->isNotEmpty()) { + return; + } + $pv = $this->pendingVolume; + + if (is_null($pv->volume_id)) { + $validator->errors()->add('id', 'A volume must be created from the pending volume first.'); + return; + } + + if ($pv->importing) { + $validator->errors()->add('id', 'An import is already in progress.'); + return; + } + + if (!$pv->import_annotations && !$pv->import_file_labels) { + $validator->errors()->add('id', 'Neither annotations nor file labels were set to be imported.'); + return; + } + + $metadata = $pv->getMetadata(); + if (is_null($metadata)) { + $validator->errors()->add('id', 'No metadata file found.'); + return; + } + + // Check file labels first because the annotations have a more expensive + // validation. + if ($pv->import_file_labels) { + $labels = $metadata->getFileLabels($pv->only_file_labels); + + if (empty($labels)) { + if ($pv->only_file_labels) { + $validator->errors()->add('id', 'There are no file labels to import with the chosen labels.'); + } else { + $validator->errors()->add('id', 'There are no file labels to import.'); + } + + return; + } + + $matchingUsers = $metadata->getMatchingUsers($pv->user_map, $pv->only_file_labels); + foreach ($matchingUsers as $id => $value) { + if (is_null($value)) { + $validator->errors()->add('id', "No matching database user could be found for metadata user ID {$id}."); + return; + } + } + + $matchingLabels = $metadata->getMatchingLabels($pv->label_map, $pv->only_file_labels); + foreach ($matchingLabels as $id => $value) { + if (is_null($value)) { + $validator->errors()->add('id', "No matching database label could be found for metadata label ID {$id}."); + return; + } + } + } + + if ($pv->import_annotations) { + $labels = $metadata->getAnnotationLabels($pv->only_annotation_labels); + + if (empty($labels)) { + if ($pv->only_annotation_labels) { + $validator->errors()->add('id', 'There are no annotations to import with the chosen labels.'); + } else { + $validator->errors()->add('id', 'There are no annotations to import.'); + } + + return; + } + + $matchingUsers = $metadata->getMatchingUsers($pv->user_map, $pv->only_annotation_labels); + foreach ($matchingUsers as $id => $value) { + if (is_null($value)) { + $validator->errors()->add('id', "No matching database user could be found for metadata user ID {$id}."); + return; + } + } + + $matchingLabels = $metadata->getMatchingLabels($pv->label_map, $pv->only_annotation_labels); + foreach ($matchingLabels as $id => $value) { + if (is_null($value)) { + $validator->errors()->add('id', "No matching database label could be found for metadata label ID {$id}."); + return; + } + } + + foreach ($metadata->getFiles() as $file) { + foreach ($file->getAnnotations() as $annotation) { + try { + $annotation->validate(); + } catch (Exception $e) { + $validator->errors()->add('id', "Invalid annotation for file {$file->name}: ".$e->getMessage()); + return; + } + } + } + } + }); + } +} diff --git a/app/Http/Requests/StoreVolume.php b/app/Http/Requests/StoreVolume.php index e466faec4..b40e1358a 100644 --- a/app/Http/Requests/StoreVolume.php +++ b/app/Http/Requests/StoreVolume.php @@ -9,16 +9,16 @@ use Biigle\Rules\VideoMetadata; use Biigle\Rules\VolumeFiles; use Biigle\Rules\VolumeUrl; -use Biigle\Traits\ParsesMetadata; +use Biigle\Services\MetadataParsing\ImageCsvParser; +use Biigle\Services\MetadataParsing\VideoCsvParser; use Biigle\Volume; -use Exception; +use File; use Illuminate\Foundation\Http\FormRequest; +use Illuminate\Http\UploadedFile; use Illuminate\Validation\Rule; class StoreVolume extends FormRequest { - use ParsesMetadata; - /** * The project to attach the new volume to. * @@ -26,6 +26,30 @@ class StoreVolume extends FormRequest */ public $project; + /** + * Class name of the metadata parser that should be used. + * + * @var string|null + */ + public $metadataParser = null; + + /** + * Filled if an uploaded metadata text was stored in a file. + * + * @var string|null + */ + protected $metadataPath; + + /** + * Remove potential temporary files. + */ + public function __destruct() + { + if (isset($this->metadataPath)) { + unlink($this->metadataPath); + } + } + /** * Determine if the user is authorized to make this request. * @@ -54,9 +78,7 @@ public function rules() 'array', ], 'handle' => ['bail', 'nullable', 'string', 'max:256', new Handle], - 'metadata_csv' => 'file|mimetypes:text/plain,text/csv,application/csv', - 'ifdo_file' => 'file', - 'metadata' => 'filled', + 'metadata_csv' => 'file|mimetypes:text/plain,text/csv,application/csv|max:500000', // Do not validate the maximum filename length with a 'files.*' rule because // this leads to a request timeout when the rule is expanded for a huge // number of files. This is checked in the VolumeFiles rule below. @@ -71,40 +93,41 @@ public function rules() */ public function withValidator($validator) { - if ($validator->fails()) { - return; - } - - // Only validate sample volume files after all other fields have been validated. $validator->after(function ($validator) { + // Only validate sample volume files after all other fields have been + // validated. + if ($validator->errors()->isNotEmpty()) { + return; + } + $files = $this->input('files'); $rule = new VolumeFiles($this->input('url'), $this->input('media_type_id')); if (!$rule->passes('files', $files)) { $validator->errors()->add('files', $rule->message()); } - if ($this->has('metadata')) { - if ($this->input('media_type_id') === MediaType::imageId()) { - $rule = new ImageMetadata($files); - } else { - $rule = new VideoMetadata($files); - } + if ($file = $this->file('metadata_csv')) { + $type = $this->input('media_type'); - if (!$rule->passes('metadata', $this->input('metadata'))) { - $validator->errors()->add('metadata', $rule->message()); + $parser = match ($type) { + 'video' => new VideoCsvParser($file), + default => new ImageCsvParser($file), + }; + + $this->metadataParser = get_class($parser); + + if (!$parser->recognizesFile()) { + $validator->errors()->add('metadata_file', 'Unknown metadata file format.'); + return; } - } - if ($this->hasFile('ifdo_file')) { - try { - // This throws an error if the iFDO is invalid. - $data = $this->parseIfdoFile($this->file('ifdo_file')); + $rule = match ($type) { + 'video' => new VideoMetadata, + default => new ImageMetadata, + }; - if ($data['media_type'] !== $this->input('media_type')) { - $validator->errors()->add('ifdo_file', 'The iFDO image-acquisition type does not match the media type of the volume.'); - } - } catch (Exception $e) { - $validator->errors()->add('ifdo_file', $e->getMessage()); + if (!$rule->passes('metadata', $parser->getMetadata())) { + $validator->errors()->add('metadata', $rule->message()); } } }); @@ -135,10 +158,13 @@ protected function prepareForValidation() $this->merge(['files' => Volume::parseFilesQueryString($files)]); } - if ($this->input('metadata_text')) { - $this->merge(['metadata' => $this->parseMetadata($this->input('metadata_text'))]); - } elseif ($this->hasFile('metadata_csv')) { - $this->merge(['metadata' => $this->parseMetadataFile($this->file('metadata_csv'))]); + if ($this->input('metadata_text') && !$this->file('metadata_csv')) { + $this->metadataPath = tempnam(sys_get_temp_dir(), 'volume_metadata'); + File::put($this->metadataPath, $this->input('metadata_text')); + $file = new UploadedFile($this->metadataPath, 'metadata.csv', 'text/csv', test: true); + // Reset this so the new file will be picked up. + unset($this->convertedFiles); + $this->files->add(['metadata_csv' => $file]); } // Backwards compatibility. diff --git a/app/Http/Requests/StoreVolumeMetadata.php b/app/Http/Requests/StoreVolumeMetadata.php index 3cc7fe2bc..df0753b2d 100644 --- a/app/Http/Requests/StoreVolumeMetadata.php +++ b/app/Http/Requests/StoreVolumeMetadata.php @@ -3,17 +3,13 @@ namespace Biigle\Http\Requests; use Biigle\Rules\ImageMetadata; -use Biigle\Rules\Utf8; use Biigle\Rules\VideoMetadata; -use Biigle\Traits\ParsesMetadata; +use Biigle\Services\MetadataParsing\ParserFactory; use Biigle\Volume; -use Exception; use Illuminate\Foundation\Http\FormRequest; class StoreVolumeMetadata extends FormRequest { - use ParsesMetadata; - /** * The volume to store the new metadata to. * @@ -28,6 +24,8 @@ class StoreVolumeMetadata extends FormRequest */ public function authorize() { + $this->volume = Volume::findOrFail($this->route('id')); + return $this->user()->can('update', $this->volume); } @@ -38,41 +36,24 @@ public function authorize() */ public function rules() { + $type = $this->volume->isImageVolume() ? 'image' : 'video'; + $parserClass = $this->input('parser', false); + $mimeTypes = []; + if ($parserClass && ParserFactory::has($type, $parserClass)) { + $mimeTypes = $parserClass::getKnownMimeTypes(); + } + return [ - 'metadata_csv' => [ - 'bail', - 'required_without_all:metadata_text,ifdo_file', + 'parser' => 'required', + 'file' => [ + 'required', 'file', - 'mimetypes:text/plain,text/csv,application/csv', - new Utf8, + 'max:500000', + 'mimetypes:'.implode(',', $mimeTypes), ], - 'metadata_text' => 'required_without_all:metadata_csv,ifdo_file', - 'ifdo_file' => 'required_without_all:metadata_csv,metadata_text|file', - 'metadata' => 'filled', ]; } - /** - * Prepare the data for validation. - * - * @return void - */ - protected function prepareForValidation() - { - $this->volume = Volume::findOrFail($this->route('id')); - - // Backwards compatibility. - if ($this->hasFile('file') && !$this->hasFile('metadata_csv')) { - $this->convertedFiles['metadata_csv'] = $this->file('file'); - } - - if ($this->hasFile('metadata_csv')) { - $this->merge(['metadata' => $this->parseMetadataFile($this->file('metadata_csv'))]); - } elseif ($this->input('metadata_text')) { - $this->merge(['metadata' => $this->parseMetadata($this->input('metadata_text'))]); - } - } - /** * Configure the validator instance. * @@ -81,36 +62,32 @@ protected function prepareForValidation() */ public function withValidator($validator) { - if ($validator->fails()) { - return; - } - $validator->after(function ($validator) { - if ($this->has('metadata')) { - $files = $this->volume->files()->pluck('filename')->toArray(); + if ($validator->errors()->isNotEmpty()) { + return; + } - if ($this->volume->isImageVolume()) { - $rule = new ImageMetadata($files); - } else { - $rule = new VideoMetadata($files); - } + $type = $this->volume->isImageVolume() ? 'image' : 'video'; + $parserClass = $this->input('parser'); + + if (!ParserFactory::has($type, $parserClass)) { + $validator->errors()->add('parser', 'Unknown metadata parser for this media type.'); + return; + } - if (!$rule->passes('metadata', $this->input('metadata'))) { - $validator->errors()->add('metadata', $rule->message()); - } + $parser = new $parserClass($this->file('file')); + if (!$parser->recognizesFile()) { + $validator->errors()->add('file', 'Unknown metadata file format.'); + return; } - if ($this->hasFile('ifdo_file')) { - try { - // This throws an error if the iFDO is invalid. - $data = $this->parseIfdoFile($this->file('ifdo_file')); + $rule = match ($type) { + 'video' => new VideoMetadata, + default => new ImageMetadata, + }; - if ($data['media_type'] !== $this->volume->mediaType->name) { - $validator->errors()->add('ifdo_file', 'The iFDO image-acquisition type does not match the media type of the volume.'); - } - } catch (Exception $e) { - $validator->errors()->add('ifdo_file', $e->getMessage()); - } + if (!$rule->passes('file', $parser->getMetadata())) { + $validator->errors()->add('file', $rule->message()); } }); } diff --git a/app/Http/Requests/UpdatePendingVolume.php b/app/Http/Requests/UpdatePendingVolume.php new file mode 100644 index 000000000..1736e0aa1 --- /dev/null +++ b/app/Http/Requests/UpdatePendingVolume.php @@ -0,0 +1,81 @@ +pendingVolume = PendingVolume::findOrFail($this->route('id')); + + return $this->user()->can('update', $this->pendingVolume); + } + + /** + * Get the validation rules that apply to the request. + * + * @return array|string> + */ + public function rules(): array + { + return [ + 'name' => 'required|max:512', + 'url' => ['required', 'string', 'max:256', new VolumeUrl], + 'files' => ['required', 'array', 'min:1'], + 'handle' => ['nullable', 'max:256', new Handle], + 'import_annotations' => 'bool', + 'import_file_labels' => 'bool', + // Do not validate the maximum filename length with a 'files.*' rule because + // this leads to a request timeout when the rule is expanded for a huge + // number of files. This is checked in the VolumeFiles rule below. + ]; + } + + /** + * Configure the validator instance. + * + * @param \Illuminate\Validation\Validator $validator + * @return void + */ + public function withValidator($validator) + { + $validator->after(function ($validator) { + // Only validate sample volume files after all other fields have been + // validated. + if ($validator->errors()->isNotEmpty()) { + return; + } + + $files = $this->input('files'); + $rule = new VolumeFiles($this->input('url'), $this->pendingVolume->media_type_id); + if (!$rule->passes('files', $files)) { + $validator->errors()->add('files', $rule->message()); + } + }); + } + + /** + * Prepare the data for validation. + * + * @return void + */ + protected function prepareForValidation() + { + $files = $this->input('files'); + if (!is_array($files)) { + $files = explode(',', $files); + } + + $files = array_map(fn ($f) => trim($f, " \n\r\t\v\x00'\""), $files); + $this->merge(['files' => array_filter($files)]); + } +} diff --git a/app/Http/Requests/UpdatePendingVolumeAnnotationLabels.php b/app/Http/Requests/UpdatePendingVolumeAnnotationLabels.php new file mode 100644 index 000000000..644544e21 --- /dev/null +++ b/app/Http/Requests/UpdatePendingVolumeAnnotationLabels.php @@ -0,0 +1,66 @@ +pendingVolume = PendingVolume::findOrFail($this->route('id')); + + return $this->user()->can('update', $this->pendingVolume); + } + + /** + * Get the validation rules that apply to the request. + * + * @return array|string> + */ + public function rules(): array + { + return [ + 'labels' => 'required|array|min:1', + ]; + } + + /** + * Configure the validator instance. + * + * @param \Illuminate\Validation\Validator $validator + * @return void + */ + public function withValidator($validator) + { + $validator->after(function ($validator) { + if ($validator->errors()->isNotEmpty()) { + return; + } + + if (is_null($this->pendingVolume->volume_id)) { + $validator->errors()->add('labels', 'A volume must be created from the pending volume first.'); + return; + } + + $labels = $this->input('labels'); + $metadata = $this->pendingVolume->getMetadata(); + if (is_null($metadata)) { + $validator->errors()->add('labels', 'No metadata file found.'); + return; + } + + $metaLabels = $metadata->getAnnotationLabels(); + foreach ($labels as $id) { + if (!array_key_exists($id, $metaLabels)) { + $validator->errors()->add('labels', "Label ID {$id} does not exist in the metadata file."); + return; + } + } + }); + } +} diff --git a/app/Http/Requests/UpdatePendingVolumeFileLabels.php b/app/Http/Requests/UpdatePendingVolumeFileLabels.php new file mode 100644 index 000000000..df89040db --- /dev/null +++ b/app/Http/Requests/UpdatePendingVolumeFileLabels.php @@ -0,0 +1,66 @@ +pendingVolume = PendingVolume::findOrFail($this->route('id')); + + return $this->user()->can('update', $this->pendingVolume); + } + + /** + * Get the validation rules that apply to the request. + * + * @return array|string> + */ + public function rules(): array + { + return [ + 'labels' => 'required|array|min:1', + ]; + } + + /** + * Configure the validator instance. + * + * @param \Illuminate\Validation\Validator $validator + * @return void + */ + public function withValidator($validator) + { + $validator->after(function ($validator) { + if ($validator->errors()->isNotEmpty()) { + return; + } + + if (is_null($this->pendingVolume->volume_id)) { + $validator->errors()->add('labels', 'A volume must be created from the pending volume first.'); + return; + } + + $labels = $this->input('labels'); + $metadata = $this->pendingVolume->getMetadata(); + if (is_null($metadata)) { + $validator->errors()->add('labels', 'No metadata file found.'); + return; + } + + $metaLabels = $metadata->getFileLabels(); + foreach ($labels as $id) { + if (!array_key_exists($id, $metaLabels)) { + $validator->errors()->add('labels', "Label ID {$id} does not exist in the metadata file."); + return; + } + } + }); + } +} diff --git a/app/Http/Requests/UpdatePendingVolumeLabelMap.php b/app/Http/Requests/UpdatePendingVolumeLabelMap.php new file mode 100644 index 000000000..b384ac48c --- /dev/null +++ b/app/Http/Requests/UpdatePendingVolumeLabelMap.php @@ -0,0 +1,101 @@ +pendingVolume = PendingVolume::findOrFail($this->route('id')); + + return $this->user()->can('update', $this->pendingVolume); + } + + /** + * Get the validation rules that apply to the request. + * + * @return array|string> + */ + public function rules(): array + { + return [ + 'label_map' => 'required|array|min:1', + 'label_map.*' => 'int', + ]; + } + + /** + * Configure the validator instance. + * + * @param \Illuminate\Validation\Validator $validator + * @return void + */ + public function withValidator($validator) + { + $validator->after(function ($validator) { + if ($validator->errors()->isNotEmpty()) { + return; + } + + if (is_null($this->pendingVolume->volume_id)) { + $validator->errors()->add('label_map', 'A volume must be created from the pending volume first.'); + return; + } + + $metadata = $this->pendingVolume->getMetadata(); + if (is_null($metadata)) { + $validator->errors()->add('label_map', 'No metadata file found.'); + return; + } + + $map = $this->input('label_map'); + $metaLabels = $metadata->getFileLabels() + $metadata->getAnnotationLabels(); + foreach ($map as $id => $dbId) { + if (!array_key_exists($id, $metaLabels)) { + $validator->errors()->add('label_map', "Label ID {$id} does not exist in the metadata file."); + return; + } + } + + $onlyLabels = $this->pendingVolume->only_annotation_labels + $this->pendingVolume->only_file_labels; + if (!empty($onlyLabels)) { + $diff = array_diff(array_keys($map), $onlyLabels); + if (!empty($diff)) { + $validator->errors()->add('label_map', 'Some chosen metadata labels were excluded by a previously defined subset of annotation and/or file labels to import.'); + } + } + + $count = Label::whereIn('id', array_values($map))->count(); + if (count($map) !== $count) { + $validator->errors()->add('label_map', 'Some label IDs do not exist in the database.'); + } + + $count = Label::whereIn('id', array_values($map)) + ->whereIn('label_tree_id', function ($query) { + // All public and all accessible private label trees. + $query->select('id') + ->from('label_trees') + ->where('visibility_id', Visibility::publicId()) + ->union( + DB::table('label_tree_user') + ->select('label_tree_id as id') + ->where('user_id', $this->user()->id) + ); + }) + ->count(); + + if (count($map) !== $count) { + $validator->errors()->add('label_map', 'You do not have access to some label IDs in the database.'); + } + }); + } +} diff --git a/app/Http/Requests/UpdatePendingVolumeUserMap.php b/app/Http/Requests/UpdatePendingVolumeUserMap.php new file mode 100644 index 000000000..a57d44851 --- /dev/null +++ b/app/Http/Requests/UpdatePendingVolumeUserMap.php @@ -0,0 +1,73 @@ +pendingVolume = PendingVolume::findOrFail($this->route('id')); + + return $this->user()->can('update', $this->pendingVolume); + } + + /** + * Get the validation rules that apply to the request. + * + * @return array|string> + */ + public function rules(): array + { + return [ + 'user_map' => 'required|array|min:1', + 'user_map.*' => 'int', + ]; + } + + /** + * Configure the validator instance. + * + * @param \Illuminate\Validation\Validator $validator + * @return void + */ + public function withValidator($validator) + { + $validator->after(function ($validator) { + if ($validator->errors()->isNotEmpty()) { + return; + } + + if (is_null($this->pendingVolume->volume_id)) { + $validator->errors()->add('user_map', 'A volume must be created from the pending volume first.'); + return; + } + + $metadata = $this->pendingVolume->getMetadata(); + if (is_null($metadata)) { + $validator->errors()->add('user_map', 'No metadata file found.'); + return; + } + + $map = $this->input('user_map'); + $metaUsers = $metadata->getUsers(); + foreach ($map as $id => $dbId) { + if (!array_key_exists($id, $metaUsers)) { + $validator->errors()->add('user_map', "User ID {$id} does not exist in the metadata file."); + return; + } + } + + $count = User::whereIn('id', array_values($map))->count(); + if (count($map) !== $count) { + $validator->errors()->add('user_map', 'Some user IDs do not exist in the database.'); + } + }); + } +} diff --git a/app/Image.php b/app/Image.php index 62841fa36..926aa6aaa 100644 --- a/app/Image.php +++ b/app/Image.php @@ -16,7 +16,7 @@ class Image extends VolumeFile /** * Allowed image MIME types. * - * @var array + * @var array */ const MIMES = [ 'image/jpeg', @@ -25,6 +25,22 @@ class Image extends VolumeFile 'image/webp', ]; + /** + * The attributes that are mass assignable. + * + * @var array + */ + protected $fillable = [ + 'filename', + 'volume_id', + 'uuid', + 'taken_at', + 'lng', + 'lat', + 'attrs', + 'tiled', + ]; + /** * The attributes hidden in the model's JSON form. * diff --git a/app/Jobs/CloneImagesOrVideos.php b/app/Jobs/CloneImagesOrVideos.php index e4b988943..aa60e2b51 100644 --- a/app/Jobs/CloneImagesOrVideos.php +++ b/app/Jobs/CloneImagesOrVideos.php @@ -9,7 +9,6 @@ use Biigle\ImageAnnotationLabel; use Biigle\ImageLabel; use Biigle\Project; -use Biigle\Traits\ChecksMetadataStrings; use Biigle\Video; use Biigle\VideoAnnotation; use Biigle\VideoAnnotationLabel; @@ -24,7 +23,7 @@ class CloneImagesOrVideos extends Job implements ShouldQueue { - use InteractsWithQueue, SerializesModels, ChecksMetadataStrings; + use InteractsWithQueue, SerializesModels; /** @@ -138,9 +137,8 @@ public function handle() ProcessNewVolumeFiles::dispatch($copy); } - //save ifdo-file if exist - if ($volume->hasIfdo()) { - $this->copyIfdoFile($volume->id, $copy->id); + if ($volume->hasMetadata()) { + $this->copyMetadataFile($volume, $copy); } $copy->creating_async = false; @@ -378,7 +376,6 @@ private function copyVideoAnnotation($volume, $copy, $selectedFileIds, $selected } collect($insertData)->chunk($parameterLimit)->each(fn ($chunk) => VideoAnnotation::insert($chunk->toArray())); - // Get the IDs of all newly inserted annotations. Ordering is essential. $newAnnotationIds = VideoAnnotation::whereIn('video_id', $chunkNewVideoIds) ->orderBy('id') @@ -443,16 +440,10 @@ private function copyVideoLabels($volume, $copy, $selectedFileIds, $selectedLabe }); } - /** Copies ifDo-Files from given volume to volume copy. - * - * @param int $volumeId - * @param int $copyId - **/ - private function copyIfdoFile($volumeId, $copyId) + private function copyMetadataFile(Volume $source, Volume $target): void { - $disk = Storage::disk(config('volumes.ifdo_storage_disk')); - $iFdoFilename = $volumeId.".yaml"; - $copyIFdoFilename = $copyId.".yaml"; - $disk->copy($iFdoFilename, $copyIFdoFilename); + $disk = Storage::disk(config('volumes.metadata_storage_disk')); + // The target metadata file path was updated in the controller method. + $disk->copy($source->metadata_file_path, $target->metadata_file_path); } } diff --git a/app/Jobs/CreateNewImagesOrVideos.php b/app/Jobs/CreateNewImagesOrVideos.php index 7280b7a87..4f43c9216 100644 --- a/app/Jobs/CreateNewImagesOrVideos.php +++ b/app/Jobs/CreateNewImagesOrVideos.php @@ -3,11 +3,9 @@ namespace Biigle\Jobs; use Biigle\Image; -use Biigle\Rules\ImageMetadata; -use Biigle\Traits\ChecksMetadataStrings; +use Biigle\Services\MetadataParsing\VolumeMetadata; use Biigle\Video; use Biigle\Volume; -use Carbon\Carbon; use DB; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Queue\InteractsWithQueue; @@ -16,7 +14,7 @@ class CreateNewImagesOrVideos extends Job implements ShouldQueue { - use InteractsWithQueue, SerializesModels, ChecksMetadataStrings; + use InteractsWithQueue, SerializesModels; /** * The volume to create the files for. @@ -44,15 +42,13 @@ class CreateNewImagesOrVideos extends Job implements ShouldQueue * * @param Volume $volume The volume to create the files for. * @param array $filenames The filenames of the files to create. - * @param array $metadata File metadata (one row per file plus column headers). * * @return void */ - public function __construct(Volume $volume, array $filenames, $metadata = []) + public function __construct(Volume $volume, array $filenames) { $this->volume = $volume; $this->filenames = $filenames; - $this->metadata = $metadata; } /** @@ -67,17 +63,16 @@ public function handle() DB::transaction(function () { $chunks = collect($this->filenames)->chunk(1000); + $metadata = $this->volume->getMetadata(); if ($this->volume->isImageVolume()) { - $metadataMap = $this->generateImageMetadataMap(); - $chunks->each(function ($chunk) use ($metadataMap) { - Image::insert($this->createFiles($chunk->toArray(), $metadataMap)); - }); + $chunks->each( + fn ($chunk) => Image::insert($this->createFiles($chunk->toArray(), $metadata)) + ); } else { - $metadataMap = $this->generateVideoMetadataMap(); - $chunks->each(function ($chunk) use ($metadataMap) { - Video::insert($this->createFiles($chunk->toArray(), $metadataMap)); - }); + $chunks->each( + fn ($chunk) => Video::insert($this->createFiles($chunk->toArray(), $metadata)) + ); } }); @@ -107,142 +102,45 @@ public function handle() /** * Create an array to be inserted as new image or video models. - * - * @param array $filenames New image/video filenames. - * @param \Illuminate\Support\Collection $metadataMap - * - * @return array */ - protected function createFiles($filenames, $metadataMap) + protected function createFiles(array $filenames, ?VolumeMetadata $metadata): array { - return array_map(function ($filename) use ($metadataMap) { - // This makes sure that the inserts have the same number of columns even if - // some images have additional metadata and others not. - $insert = array_fill_keys( - array_merge(['attrs'], ImageMetadata::ALLOWED_ATTRIBUTES), - null - ); + $metaKeys = []; + $insertData = []; - $insert = array_merge($insert, [ - 'filename' => $filename, - 'volume_id' => $this->volume->id, - 'uuid' => (string) Uuid::uuid4(), - ]); + foreach ($filenames as $filename) { + $insert = []; - $metadata = collect($metadataMap->get($filename)); - if ($metadata->isNotEmpty()) { - // Remove empty cells. - $metadata = $metadata->filter(); - $insert = array_merge( - $insert, - $metadata->only(ImageMetadata::ALLOWED_ATTRIBUTES)->toArray() - ); + if ($metadata && ($fileMeta = $metadata->getFile($filename))) { + $insert = array_map(function ($item) { + if (is_array($item)) { + return json_encode($item); + } - $more = $metadata->only(ImageMetadata::ALLOWED_METADATA); - if ($more->isNotEmpty()) { - $insert['attrs'] = collect(['metadata' => $more])->toJson(); - } + return $item; + }, $fileMeta->getInsertData()); } - return $insert; - }, $filenames); - } - /** - * Generate a map for image metadata that is indexed by filename. - * - * @return \Illuminate\Support\Collection - */ - protected function generateImageMetadataMap() - { - if (empty($this->metadata)) { - return collect([]); - } - - $columns = $this->metadata[0]; - - $map = collect(array_slice($this->metadata, 1)) - ->map(fn ($row) => array_combine($columns, $row)) - ->map(function ($row) { - if (array_key_exists('taken_at', $row)) { - $row['taken_at'] = Carbon::parse($row['taken_at']); - } + $metaKeys = array_merge($metaKeys, array_keys($insert)); - return $row; - }) - ->keyBy('filename'); - - $map->forget('filename'); - - return $map; - } + $insert = array_merge($insert, [ + 'filename' => $filename, + 'volume_id' => $this->volume->id, + 'uuid' => (string) Uuid::uuid4(), + ]); - /** - * Generate a map for video metadata that is indexed by filename. - * - * @return \Illuminate\Support\Collection - */ - protected function generateVideoMetadataMap() - { - if (empty($this->metadata)) { - return collect([]); + $insertData[] = $insert; } - $columns = $this->metadata[0]; - - $map = collect(array_slice($this->metadata, 1)) - ->map(fn ($row) => array_combine($columns, $row)) - ->map(function ($row) { - if (array_key_exists('taken_at', $row)) { - $row['taken_at'] = Carbon::parse($row['taken_at']); - } else { - $row['taken_at'] = null; - } - - return $row; - }) - ->sortBy('taken_at') - ->groupBy('filename') - ->map(fn ($entries) => $this->processVideoColumns($entries, $columns)); + $metaKeys = array_unique($metaKeys); - return $map; - } - - /** - * Generate the metadata map entry for a single video file. - * - * @param \Illuminate\Support\Collection $entries - * @param \Illuminate\Support\Collection $columns - * - * @return \Illuminate\Support\Collection - */ - protected function processVideoColumns($entries, $columns) - { - $return = collect([]); - foreach ($columns as $column) { - $values = $entries->pluck($column); - if ($values->filter([$this, 'isFilledString'])->isEmpty()) { - // Ignore completely empty columns. - continue; - } - - $return[$column] = $values; - - if (in_array($column, array_keys(ImageMetadata::NUMERIC_FIELDS))) { - $return[$column] = $return[$column]->map(function ($x) { - // This check is required since floatval would return 0 for - // an empty value. This could skew metadata. - return $this->isFilledString($x) ? floatval($x) : null; - }); - } - - if (in_array($column, ImageMetadata::ALLOWED_ATTRIBUTES)) { - $return[$column] = $return[$column]->toJson(); - } + // Ensure that each item has the same keys even if some are missing metadata. + if (!empty($metaKeys)) { + $fill = array_fill_keys($metaKeys, null); + $insertData = array_map(fn ($i) => array_merge($fill, $i), $insertData); } - $return->forget('filename'); - - return $return; + return $insertData; } } diff --git a/app/Jobs/ImportVolumeMetadata.php b/app/Jobs/ImportVolumeMetadata.php new file mode 100644 index 000000000..4506c5bf5 --- /dev/null +++ b/app/Jobs/ImportVolumeMetadata.php @@ -0,0 +1,215 @@ +pv->volume->creating_async) { + // Wait 10 minutes so the volume has a chance to finish creating the files. + // Do this for a maximum of 120 min (12 tries). + $this->release(600); + + return; + } + + DB::transaction(function () { + $metadata = $this->pv->getMetadata(); + + $annotationUserMap = $metadata->getMatchingUsers($this->pv->user_map, $this->pv->only_annotation_labels); + $annotationLabelMap = $metadata->getMatchingLabels($this->pv->label_map, $this->pv->only_annotation_labels); + + $fileLabelUserMap = $metadata->getMatchingUsers($this->pv->user_map, $this->pv->only_file_labels); + $fileLabelLabelMap = $metadata->getMatchingLabels($this->pv->label_map, $this->pv->only_file_labels); + + foreach ($this->pv->volume->files()->lazyById() as $file) { + $metaFile = $metadata->getFile($file->filename); + if (!$metaFile) { + continue; + } + + if ($this->pv->import_annotations && $metaFile->hasAnnotations()) { + $this->insertAnnotations($metaFile, $file, $annotationUserMap, $annotationLabelMap); + } + + if ($this->pv->import_file_labels && $metaFile->hasFileLabels()) { + $this->insertFileLabels($metaFile, $file, $fileLabelUserMap, $fileLabelLabelMap); + } + } + }); + + $this->pv->delete(); + } + + /** + * Insert metadata annotations of a file into the database. + */ + protected function insertAnnotations( + FileMetadata $meta, + VolumeFile $file, + array $userMap, + array $labelMap + ): void { + $insertAnnotations = []; + $insertAnnotationLabels = []; + $now = now()->toDateTimeString(); + + foreach ($meta->getAnnotations() as $index => $annotation) { + // This will remove labels that should be ignored based on $onlyLabels and + // that have no match in the database. + $annotationLabels = array_filter( + $annotation->labels, + fn ($lau) => !is_null($labelMap[$lau->label->id] ?? null) + ); + + if (empty($annotationLabels)) { + continue; + } + + $insertAnnotations[] = array_merge($annotation->getInsertData($file->id), [ + 'created_at' => $now, + 'updated_at' => $now, + ]); + + $insertAnnotationLabels[] = array_map(fn ($lau) => [ + 'label_id' => $labelMap[$lau->label->id], + 'user_id' => $userMap[$lau->user->id], + 'created_at' => $now, + 'updated_at' => $now, + ], $annotationLabels); + + // Insert in chunks because a single file can have tens of thousands of + // annotations (e.g. a video or mosaic). + if (($index % static::$insertChunkSize) === 0) { + $this->insertAnnotationChunk($file, $insertAnnotations, $insertAnnotationLabels); + $insertAnnotations = []; + $insertAnnotationLabels = []; + } + } + + if (!empty($insertAnnotations)) { + $this->insertAnnotationChunk($file, $insertAnnotations, $insertAnnotationLabels); + } + } + + protected function insertAnnotationChunk( + VolumeFile $file, + array $annotations, + array $annotationLabels + ): void { + $file->annotations()->insert($annotations); + + $ids = $file->annotations() + ->orderBy('id', 'desc') + ->take(count($annotations)) + ->pluck('id') + ->reverse() + ->toArray(); + + foreach ($ids as $index => $id) { + foreach ($annotationLabels[$index] as &$i) { + $i['annotation_id'] = $id; + } + } + + // Flatten. Use array_values to prevent accidental array unpacking with string + // keys (which makes the linter complain). + $annotationLabels = array_merge(...array_values($annotationLabels)); + + if ($file instanceof Image) { + foreach ($annotationLabels as &$i) { + $i['confidence'] = 1.0; + } + + ImageAnnotationLabel::insert($annotationLabels); + } else { + VideoAnnotationLabel::insert($annotationLabels); + } + } + + /** + * Insert metadata file labels of a file into the database. + */ + protected function insertFileLabels( + FileMetadata $meta, + VolumeFile $file, + array $userMap, + array $labelMap + ): void { + // This will remove labels that should be ignored based on $onlyLabels and + // that have no match in the database. + $fileLabels = array_filter( + $meta->getFileLabels(), + fn ($lau) => !is_null($labelMap[$lau->label->id] ?? null) + ); + + if (empty($fileLabels)) { + return; + } + + $insertFileLabels = array_map(fn ($lau) => [ + 'label_id' => $labelMap[$lau->label->id], + 'user_id' => $userMap[$lau->user->id], + ], $fileLabels); + + if ($file instanceof Image) { + foreach ($insertFileLabels as &$i) { + $i['image_id'] = $file->id; + } + + ImageLabel::insert($insertFileLabels); + } else { + foreach ($insertFileLabels as &$i) { + $i['video_id'] = $file->id; + } + + VideoLabel::insert($insertFileLabels); + } + } +} diff --git a/app/Jobs/UpdateVolumeMetadata.php b/app/Jobs/UpdateVolumeMetadata.php new file mode 100644 index 000000000..c884375ad --- /dev/null +++ b/app/Jobs/UpdateVolumeMetadata.php @@ -0,0 +1,68 @@ +volume->getMetadata(); + + if (!$metadata) { + return; + } + + foreach ($this->volume->files()->lazyById() as $file) { + $fileMeta = $metadata->getFile($file->filename); + if (!$fileMeta) { + continue; + } + + $insert = $fileMeta->getInsertData(); + + // If a video is updated with timestamped metadata, the old metadata must + // be replaced entirely. + if (($file instanceof Video) && array_key_exists('taken_at', $insert)) { + $file->taken_at = null; + $file->lat = null; + $file->lng = null; + $file->metadata = null; + } + + $attrs = $insert['attrs'] ?? null; + unset($insert['attrs']); + $file->fill($insert); + if ($attrs) { + $file->metadata = array_merge($file->metadata ?: [], $attrs['metadata']); + } + + if ($file->isDirty()) { + $file->save(); + } + } + + $this->volume->flushGeoInfoCache(); + } +} diff --git a/app/Observers/VolumeObserver.php b/app/Observers/VolumeObserver.php index 397798cb1..94e8deb70 100644 --- a/app/Observers/VolumeObserver.php +++ b/app/Observers/VolumeObserver.php @@ -48,7 +48,7 @@ public function deleting(Volume $volume) event(new VideosDeleted($uuids)); } - $volume->deleteIfdo(); + $volume->deleteMetadata(true); return true; } diff --git a/app/PendingVolume.php b/app/PendingVolume.php new file mode 100644 index 000000000..9820d6144 --- /dev/null +++ b/app/PendingVolume.php @@ -0,0 +1,86 @@ + + */ + protected $fillable = [ + 'media_type_id', + 'user_id', + 'project_id', + 'metadata_file_path', + 'metadata_parser', + 'volume_id', + 'import_annotations', + 'import_file_labels', + 'only_annotation_labels', + 'only_file_labels', + 'label_map', + 'user_map', + 'importing', + ]; + + /** + * The attributes that should be hidden for arrays. + * + * @var array + */ + protected $hidden = [ + 'metadata_file_path', + ]; + + /** + * The attributes that should be cast. + * + * @var array + */ + protected $casts = [ + 'only_annotation_labels' => 'array', + 'only_file_labels' => 'array', + 'label_map' => 'array', + 'user_map' => 'array', + ]; + + /** + * Default values for attributes. + * + * @var array + */ + protected $attributes = [ + 'only_annotation_labels' => '[]', + 'only_file_labels' => '[]', + 'label_map' => '[]', + 'user_map' => '[]', + ]; + + protected static function booted(): void + { + static::$metadataFileDisk = config('volumes.pending_metadata_storage_disk'); + + static::deleting(function (PendingVolume $pv) { + $pv->deleteMetadata(true); + }); + } + + public function project(): BelongsTo + { + return $this->belongsTo(Project::class); + } + + public function volume(): BelongsTo + { + return $this->belongsTo(Volume::class); + } +} diff --git a/app/Policies/PendingVolumePolicy.php b/app/Policies/PendingVolumePolicy.php new file mode 100644 index 000000000..078c758b7 --- /dev/null +++ b/app/Policies/PendingVolumePolicy.php @@ -0,0 +1,61 @@ +can('sudo')) { + return true; + } + } + + /** + * Determine if the given pending volume can be accessed by the user. + */ + public function access(User $user, PendingVolume $pv): bool + { + return $user->id === $pv->user_id && + $this->remember( + "pending-volume-can-access-{$user->id}-{$pv->id}", + fn () => + DB::table('project_user') + ->where('project_id', $pv->project_id) + ->where('user_id', $user->id) + ->where('project_role_id', Role::adminId()) + ->exists() + ); + } + + /** + * Determine if the given pending volume can be updated by the user. + */ + public function update(User $user, PendingVolume $pv): bool + { + return $this->access($user, $pv); + } + + /** + * Determine if the given pending volume can be deleted by the user. + */ + public function destroy(User $user, PendingVolume $pv): bool + { + return $this->access($user, $pv); + } +} diff --git a/app/Project.php b/app/Project.php index d314b7d58..0fa35a528 100644 --- a/app/Project.php +++ b/app/Project.php @@ -209,6 +209,16 @@ public function videoVolumes() return $this->volumes()->where('media_type_id', MediaType::videoId()); } + /** + * The pending volumes of this project. + * + * @return \Illuminate\Database\Eloquent\Relations\HasMany + */ + public function pendingVolumes() + { + return $this->hasMany(PendingVolume::class); + } + /** * Adds a volume to this project if it wasn't already. * diff --git a/app/Rules/ImageMetadata.php b/app/Rules/ImageMetadata.php index 6d012c006..7bf789dca 100644 --- a/app/Rules/ImageMetadata.php +++ b/app/Rules/ImageMetadata.php @@ -2,33 +2,12 @@ namespace Biigle\Rules; +use Biigle\Services\MetadataParsing\FileMetadata; +use Biigle\Services\MetadataParsing\VolumeMetadata; use Illuminate\Contracts\Validation\Rule; class ImageMetadata implements Rule { - /** - * Allowed columns for the metadata information to change image attributes. - * - * @var array - */ - const ALLOWED_ATTRIBUTES = [ - 'lat', - 'lng', - 'taken_at', - ]; - - /** - * Allowed columns for the metadata information to change image metadata. - * - * @var array - */ - const ALLOWED_METADATA = [ - 'area', - 'distance_to_ground', - 'gps_altitude', - 'yaw', - ]; - /** * All numeric metadata fields (keys) with description (values). * @@ -36,20 +15,13 @@ class ImageMetadata implements Rule */ const NUMERIC_FIELDS = [ 'area' => 'area', - 'distance_to_ground' => 'distance to ground', - 'gps_altitude' => 'GPS altitude', + 'distanceToGround' => 'distance to ground', + 'gpsAltitude' => 'GPS altitude', 'lat' => 'latitude', 'lng' => 'longitude', 'yaw' => 'yaw', ]; - /** - * Array of volume file names. - * - * @var array - */ - protected $files; - /** * The validation error message. * @@ -59,176 +31,104 @@ class ImageMetadata implements Rule /** * Create a new instance. - * - * @param array $files */ - public function __construct($files) + public function __construct() { - $this->files = $files; $this->message = "The :attribute is invalid."; } /** * Determine if the validation rule passes. - * - * @param string $attribute - * @param array $value - * @return bool */ - public function passes($attribute, $value) + public function passes($attribute, $value): bool { - if (!is_array($value)) { - return false; + if (!($value instanceof VolumeMetadata)) { + throw new \Exception('No value of type '.VolumeMetadata::class.' given.'); } // This checks if any information is given at all. - if (empty($value)) { + if ($value->isEmpty()) { $this->message = 'The metadata information is empty.'; return false; } - $columns = array_shift($value); - - // This checks if any information is given beside the column description. - if (empty($value)) { - $this->message = 'The metadata information is empty.'; + $fileMetadata = $value->getFiles(); - return false; - } - - if (!in_array('filename', $columns)) { - $this->message = 'The filename column is required.'; - - return false; - } - - $colCount = count($columns); - - if ($colCount === 1) { - $this->message = 'No metadata columns given.'; - - return false; - } - - if ($colCount !== count(array_unique($columns))) { - $this->message = 'Each column may occur only once.'; - - return false; - } - - $allowedColumns = array_merge(['filename'], self::ALLOWED_ATTRIBUTES, self::ALLOWED_METADATA); - $diff = array_diff($columns, $allowedColumns); - - if (count($diff) > 0) { - $this->message = 'The columns array may contain only values of: '.implode(', ', $allowedColumns).'.'; - - return false; + foreach ($fileMetadata as $file) { + if (!$this->fileMetadataPasses($file)) { + return false; + } } - $lng = in_array('lng', $columns); - $lat = in_array('lat', $columns); - if ($lng && !$lat || !$lng && $lat) { - $this->message = "If the 'lng' column is present, the 'lat' column must be present, too (and vice versa)."; - - return false; - } + return true; + } - foreach ($value as $index => $row) { - // +1 since index starts at 0. - // +1 since column description row was removed above. - $line = $index + 2; + /** + * Get the validation error message. + * + * @return string + */ + public function message() + { + return $this->message; + } - if (count($row) !== $colCount) { - $this->message = "Invalid column count in line {$line}."; + protected function fileMetadataPasses(FileMetadata $file) + { + if (!is_null($file->lng)) { + if (abs($file->lng) > 180) { + $this->message = "'{$file->lng}' is no valid longitude for file {$file->name}."; return false; } - $combined = array_combine($columns, $row); - $combined = array_filter($combined); - if (!array_key_exists('filename', $combined)) { - $this->message = "Filename missing in line {$line}."; + if (is_null($file->lat)) { + $this->message = "Missing latitude for file {$file->name}."; return false; } + } - $filename = $combined['filename']; - - if (!in_array($filename, $this->files)) { - $this->message = "There is no file with filename {$filename}."; + if (!is_null($file->lat)) { + if (abs($file->lat) > 90) { + $this->message = "'{$file->lat}' is no valid latitude for file {$file->name}."; return false; } - if (array_key_exists('lng', $combined)) { - $lng = $combined['lng']; - if (!is_numeric($lng) || abs($lng) > 180) { - $this->message = "'{$lng}' is no valid longitude for file {$filename}."; - - return false; - } - - - if (!array_key_exists('lat', $combined)) { - $this->message = "Missing latitude for file {$filename}."; - - return false; - } - } - - if (array_key_exists('lat', $combined)) { - $lat = $combined['lat']; - if (!is_numeric($lat) || abs($lat) > 90) { - $this->message = "'{$lat}' is no valid latitude for file {$filename}."; - - return false; - } + if (is_null($file->lng)) { + $this->message = "Missing longitude for file {$file->name}."; - if (!array_key_exists('lng', $combined)) { - $this->message = "Missing longitude for file {$filename}."; - - return false; - } + return false; } + } - // Catch both a malformed date (false) and the zero date (negative integer). - if (array_key_exists('taken_at', $combined)) { - $date = $combined['taken_at']; - if (!(strtotime($date) > 0)) { - $this->message = "'{$date}' is no valid date for file {$filename}."; + // Catch both a malformed date (false) and the zero date (negative integer). + if (!is_null($file->takenAt)) { + if (!(strtotime($file->takenAt) > 0)) { + $this->message = "'{$file->takenAt}' is no valid date for file {$file->name}."; - return false; - } + return false; } + } - foreach (self::NUMERIC_FIELDS as $key => $text) { - if (array_key_exists($key, $combined) && !is_numeric($combined[$key])) { - $this->message = "'{$combined[$key]}' is no valid {$text} for file {$filename}."; + foreach (self::NUMERIC_FIELDS as $key => $text) { + if (!is_null($file->$key) && !is_numeric($file->$key)) { + $this->message = "'{$file->$key}' is no valid {$text} for file {$file->name}."; - return false; - } + return false; } + } - if (array_key_exists('yaw', $combined)) { - if ($combined['yaw'] < 0 || $combined['yaw'] > 360) { - $this->message = "'{$combined['yaw']}' is no valid yaw for file {$filename}."; + if (!is_null($file->yaw)) { + if ($file->yaw < 0 || $file->yaw > 360) { + $this->message = "'{$file->yaw}' is no valid yaw for file {$file->name}."; - return false; - } + return false; } } return true; } - - /** - * Get the validation error message. - * - * @return string - */ - public function message() - { - return $this->message; - } } diff --git a/app/Rules/Utf8.php b/app/Rules/Utf8.php deleted file mode 100644 index f2a9bdf83..000000000 --- a/app/Rules/Utf8.php +++ /dev/null @@ -1,31 +0,0 @@ -get(); - return mb_detect_encoding($value, 'UTF-8', true) !== false; - } - - /** - * Get the validation error message. - * - * @return string - */ - public function message() - { - return "The :attribute must be UTF-8 encoded."; - } -} diff --git a/app/Rules/VideoMetadata.php b/app/Rules/VideoMetadata.php index a23c42ee9..83ad3110b 100644 --- a/app/Rules/VideoMetadata.php +++ b/app/Rules/VideoMetadata.php @@ -7,7 +7,7 @@ class VideoMetadata extends ImageMetadata /** * {@inheritdoc} */ - public function passes($attribute, $value) + public function passes($attribute, $value): bool { $passes = parent::passes($attribute, $value); @@ -15,27 +15,13 @@ public function passes($attribute, $value) return false; } - $columns = array_shift($value); - - $filenames = []; - foreach ($value as $index => $row) { - $combined = array_combine($columns, $row); - $combined = array_filter($combined); - $filename = $combined['filename']; - if (array_key_exists($filename, $filenames)) { - // If this exists, it was already checked if it is a valid date by the - // parent method. - if (!array_key_exists('taken_at', $combined)) { - // +1 since index starts at 0. - // +1 since column description row was removed above. - $line = $index + 2; - - $this->message = "File {$filename} has multiple entries but no 'taken_at' at line {$line}."; + $fileMetadata = $value->getFiles(); + foreach ($fileMetadata as $file) { + foreach ($file->getFrames() as $frame) { + if (!$this->fileMetadataPasses($frame)) { return false; } - } else { - $filenames[$filename] = true; } } diff --git a/app/Services/MetadataParsing/Annotation.php b/app/Services/MetadataParsing/Annotation.php new file mode 100644 index 000000000..45d973e66 --- /dev/null +++ b/app/Services/MetadataParsing/Annotation.php @@ -0,0 +1,65 @@ +|array> $points + * @param array $labels + */ + public function __construct( + public Shape $shape, + public array $points, + public array $labels, + ) { + $this->shape_id = $shape->id; + } + + /** + * Get the array of metadata that can be used for Model::insert(); + * + * @param int $id ID of the image/video database model. + */ + public function getInsertData(int $id): array + { + return [ + 'points' => json_encode($this->points), + 'shape_id' => $this->shape->id, + ]; + } + + /** + * Validatethe points and labels. + * + * @throws Exception If something is invalid. + */ + public function validate(): void + { + if (empty($this->labels)) { + throw new Exception('The annotation has no labels.'); + } + + $this->validatePoints($this->points); + } + + /** + * {@inheritdoc} + */ + public function setPointsAttribute(array $points) + { + $this->points = array_map(fn ($coordinate) => round($coordinate, 2), $points); + } +} diff --git a/app/Services/MetadataParsing/CsvParser.php b/app/Services/MetadataParsing/CsvParser.php new file mode 100644 index 000000000..eb11e4eda --- /dev/null +++ b/app/Services/MetadataParsing/CsvParser.php @@ -0,0 +1,119 @@ + 'filename', + 'lon' => 'lng', + 'longitude' => 'lng', + 'latitude' => 'lat', + 'heading' => 'yaw', + 'sub_datetime' => 'taken_at', + 'sub_longitude' => 'lng', + 'sub_latitude' => 'lat', + 'sub_heading' => 'yaw', + 'sub_distance' => 'distance_to_ground', + 'sub_altitude' => 'gps_altitude', + ]; + + /** + * {@inheritdoc} + */ + public static function getKnownMimeTypes(): array + { + return [ + 'text/plain', + 'text/csv', + ]; + } + + /** + * {@inheritdoc} + */ + public static function getName(): string + { + return 'BIIGLE CSV'; + } + + /** + * {@inheritdoc} + */ + public function recognizesFile(): bool + { + $file = $this->getCsvIterator(); + $line = $file->current(); + if (!is_array($line) || empty($line)) { + return false; + } + + if (mb_detect_encoding($line[0], 'UTF-8', true) === false) { + return false; + } + + $line = $this->processFirstLine($line); + + if (!in_array('filename', $line, true) && !in_array('file', $line, true)) { + return false; + } + + return true; + } + + /** + * {@inheritdoc} + */ + abstract public function getMetadata(): VolumeMetadata; + + protected function getCsvIterator(): SeekableIterator + { + $file = parent::getFileObject(); + $file->setFlags(SplFileObject::READ_CSV); + + return $file; + } + + protected function processFirstLine(array $line): array + { + $line = array_map('strtolower', $line); + if (!empty($line[0])) { + $line[0] = Util::removeBom($line[0]); + } + + return $line; + } + + protected function getKeyMap(array $line): array + { + $line = $this->processFirstLine($line); + + $keys = array_map(function ($column) { + if (array_key_exists($column, self::COLUMN_SYNONYMS)) { + return self::COLUMN_SYNONYMS[$column]; + } + + return $column; + }, $line); + + // This will remove duplicate columns and retain the "last" one. + return array_flip($keys); + } + + /** + * Cast the value to float if it is not null or an empty string. + */ + protected function maybeCastToFloat(?string $value): ?float + { + return (is_null($value) || $value === '') ? null : floatval($value); + } +} diff --git a/app/Services/MetadataParsing/FileMetadata.php b/app/Services/MetadataParsing/FileMetadata.php new file mode 100644 index 000000000..be9aa6895 --- /dev/null +++ b/app/Services/MetadataParsing/FileMetadata.php @@ -0,0 +1,154 @@ + + */ + public array $annotations = []; + + /** + * The labels directly attached to the file. + * + * @var array + */ + public array $labels = []; + + public function __construct( + public string $name, + public ?float $lat = null, + public ?float $lng = null, + public ?string $takenAt = null, + public ?float $area = null, + public ?float $distanceToGround = null, + public ?float $gpsAltitude = null, + public ?float $yaw = null, + ) { + $this->name = trim($this->name); + } + + public function isEmpty(): bool + { + return true; + } + + /** + * Get the array of metadata that can be used for Model::insert(); + */ + public function getInsertData(): array + { + return []; + } + + public function addAnnotation(Annotation $annotation): void + { + $this->annotations[] = $annotation; + } + + public function getAnnotations(): array + { + return $this->annotations; + } + + public function hasAnnotations(): bool + { + return !empty($this->annotations); + } + + public function addFileLabel(LabelAndUser $lau): void + { + $this->labels[] = $lau; + } + + public function getFileLabels(): array + { + return $this->labels; + } + + public function hasFileLabels(): bool + { + return !empty($this->labels); + } + + /** + * @return array Labels indexed by ID. + */ + public function getFileLabelLabels(array $onlyLabels = []): array + { + $labels = []; + + foreach ($this->getFileLabelLabelAndUsers($onlyLabels) as $lau) { + $labels[$lau->label->id] = $lau->label; + } + + return $labels; + } + + /** + * @return array Labels indexed by ID. + */ + public function getAnnotationLabels(array $onlyLabels = []): array + { + $labels = []; + + foreach ($this->getAnnotationLabelAndUsers($onlyLabels) as $lau) { + $labels[$lau->label->id] = $lau->label; + } + + return $labels; + } + + /** + * @param array $onlyLabels List of metadata label IDs to filter the list of users. + * + * @return array Users indexed by ID. + */ + public function getUsers(array $onlyLabels = []): array + { + $users = []; + + foreach ($this->getAnnotationLabelAndUsers($onlyLabels) as $lau) { + $users[$lau->user->id] = $lau->user; + } + + foreach ($this->getFileLabelLabelAndUsers($onlyLabels) as $lau) { + $users[$lau->user->id] = $lau->user; + } + + return $users; + } + + protected function getAnnotationLabelAndUsers(array $onlyLabels = []): array + { + $ret = []; + $onlyLabels = array_flip($onlyLabels); + + foreach ($this->getAnnotations() as $annotation) { + if (!$onlyLabels) { + $add = $annotation->labels; + } else { + $add = array_filter($annotation->labels, fn ($lau) => array_key_exists($lau->label->id, $onlyLabels)); + } + + $ret = array_merge($ret, $add); + } + + return $ret; + } + + protected function getFileLabelLabelAndUsers(array $onlyLabels = []): array + { + if (!$onlyLabels) { + return $this->getFileLabels(); + } + + $onlyLabels = array_flip($onlyLabels); + + return array_filter( + $this->getFileLabels(), + fn ($lau) => array_key_exists($lau->label->id, $onlyLabels) + ); + } +} diff --git a/app/Services/MetadataParsing/ImageAnnotation.php b/app/Services/MetadataParsing/ImageAnnotation.php new file mode 100644 index 000000000..e57e784fc --- /dev/null +++ b/app/Services/MetadataParsing/ImageAnnotation.php @@ -0,0 +1,16 @@ + $id, + ]); + } +} diff --git a/app/Services/MetadataParsing/ImageCsvParser.php b/app/Services/MetadataParsing/ImageCsvParser.php new file mode 100644 index 000000000..9859aa783 --- /dev/null +++ b/app/Services/MetadataParsing/ImageCsvParser.php @@ -0,0 +1,55 @@ +getCsvIterator(); + $line = $file->current(); + if (!is_array($line)) { + return $data; + } + + $keyMap = $this->getKeyMap($line); + + $getValue = fn ($row, $key) => $row[$keyMap[$key] ?? null] ?? null; + + $file->next(); + while ($file->valid()) { + $row = $file->current(); + $file->next(); + if (empty($row)) { + continue; + } + + $name = $getValue($row, 'filename'); + if (empty($name)) { + continue; + } + + $fileData = new ImageMetadata( + name: $getValue($row, 'filename'), + lat: $this->maybeCastToFloat($getValue($row, 'lat')), + lng: $this->maybeCastToFloat($getValue($row, 'lng')), + takenAt: $getValue($row, 'taken_at') ?: null, // Use null instead of ''. + area: $this->maybeCastToFloat($getValue($row, 'area')), + distanceToGround: $this->maybeCastToFloat($getValue($row, 'distance_to_ground')), + gpsAltitude: $this->maybeCastToFloat($getValue($row, 'gps_altitude')), + yaw: $this->maybeCastToFloat($getValue($row, 'yaw')), + ); + + $data->addFile($fileData); + } + + return $data; + } +} diff --git a/app/Services/MetadataParsing/ImageMetadata.php b/app/Services/MetadataParsing/ImageMetadata.php new file mode 100644 index 000000000..526314da1 --- /dev/null +++ b/app/Services/MetadataParsing/ImageMetadata.php @@ -0,0 +1,66 @@ +lat) + && is_null($this->lng) + && is_null($this->takenAt) + && is_null($this->area) + && is_null($this->distanceToGround) + && is_null($this->gpsAltitude) + && is_null($this->yaw); + } + + /** + * Get the array of metadata that can be used for Model::insert(); + */ + public function getInsertData(): array + { + $data = ['filename' => $this->name]; + + if (!is_null($this->lat)) { + $data['lat'] = $this->lat; + } + + if (!is_null($this->lng)) { + $data['lng'] = $this->lng; + } + + if (!is_null($this->takenAt)) { + $data['taken_at'] = Carbon::parse($this->takenAt)->toDateTimeString(); + } + + $attrs = []; + + if (!is_null($this->area)) { + $attrs['area'] = $this->area; + } + + if (!is_null($this->distanceToGround)) { + $attrs['distance_to_ground'] = $this->distanceToGround; + } + + if (!is_null($this->gpsAltitude)) { + $attrs['gps_altitude'] = $this->gpsAltitude; + } + + if (!is_null($this->yaw)) { + $attrs['yaw'] = $this->yaw; + } + + if (!empty($attrs)) { + $data['attrs'] = ['metadata' => $attrs]; + } + + return $data; + } +} diff --git a/app/Services/MetadataParsing/Label.php b/app/Services/MetadataParsing/Label.php new file mode 100644 index 000000000..2c8e13655 --- /dev/null +++ b/app/Services/MetadataParsing/Label.php @@ -0,0 +1,41 @@ + $this->id, + 'name' => $this->name, + ]; + + if (!is_null($this->color)) { + $ret['color'] = $this->color; + } + + if (!is_null($this->uuid)) { + $ret['uuid'] = $this->uuid; + } + + return $ret; + } +} diff --git a/app/Services/MetadataParsing/LabelAndUser.php b/app/Services/MetadataParsing/LabelAndUser.php new file mode 100644 index 000000000..0f96ca861 --- /dev/null +++ b/app/Services/MetadataParsing/LabelAndUser.php @@ -0,0 +1,13 @@ +fileObject)) { + $this->fileObject = $this->file->openFile(); + } + + return $this->fileObject; + } +} diff --git a/app/Services/MetadataParsing/ParserFactory.php b/app/Services/MetadataParsing/ParserFactory.php new file mode 100644 index 000000000..346c0b9b8 --- /dev/null +++ b/app/Services/MetadataParsing/ParserFactory.php @@ -0,0 +1,37 @@ + [ + ImageCsvParser::class, + ], + 'video' => [ + VideoCsvParser::class, + ], + ]; + + /** + * Check if the metadata parser exists for the given type. + */ + public static function has(string $type, string $class): bool + { + return in_array($class, static::$parsers[$type] ?? []); + } + + /** + * Add a new metadata parser to the list of known parsers. + */ + public static function extend(string $parserClass, string $type): void + { + if (!in_array(MetadataParser::class, class_parents($parserClass))) { + throw new Exception("A metadata parser must extend ".MetadataParser::class); + } + + self::$parsers[$type][] = $parserClass; + } +} diff --git a/app/Services/MetadataParsing/User.php b/app/Services/MetadataParsing/User.php new file mode 100644 index 000000000..89c6e88b0 --- /dev/null +++ b/app/Services/MetadataParsing/User.php @@ -0,0 +1,35 @@ + $this->id, + 'name' => $this->name, + ]; + + if (!is_null($this->uuid)) { + $ret['uuid'] = $this->uuid; + } + + return $ret; + } +} diff --git a/app/Services/MetadataParsing/VideoAnnotation.php b/app/Services/MetadataParsing/VideoAnnotation.php new file mode 100644 index 000000000..cd3af3d99 --- /dev/null +++ b/app/Services/MetadataParsing/VideoAnnotation.php @@ -0,0 +1,75 @@ +> $points + * @param array $labels + * @param array $frames + */ + public function __construct( + public Shape $shape, + public array $points, + public array $labels, + public array $frames, + ) { + parent::__construct($shape, $points, $labels); + } + + /** + * {@inheritdoc} + */ + public function getInsertData(int $id): array + { + return array_merge(parent::getInsertData($id), [ + 'video_id' => $id, + 'frames' => json_encode($this->frames), + ]); + } + + /** + * {@inheritdoc} + */ + public function validate(): void + { + parent::validate(); + + foreach ($this->frames as $frame) { + /** @phpstan-ignore booleanAnd.alwaysFalse */ + if (!is_float($frame) && !is_int($frame)) { + throw new Exception("Video annotation frames must be numbers, got '{$frame}'."); + } + } + } + + /** + * Similar to \Biigle\VideoAnnotation::validatePoints. + */ + public function validatePoints(array $points = []): void + { + if ($this->shape_id === Shape::wholeFrameId()) { + if (count($this->points) !== 0) { + throw new Exception('Whole frame annotations cannot have point coordinates.'); + } + + return; + } + + if (count($this->points) !== count($this->frames)) { + throw new Exception('The number of key frames does not match the number of annotation coordinates.'); + } + + // Gaps are represented as empty arrays + array_map(function ($point) { + if (count($point)) { + parent::validatePoints($point); + } + }, $this->points); + } +} diff --git a/app/Services/MetadataParsing/VideoCsvParser.php b/app/Services/MetadataParsing/VideoCsvParser.php new file mode 100644 index 000000000..23b83c89d --- /dev/null +++ b/app/Services/MetadataParsing/VideoCsvParser.php @@ -0,0 +1,74 @@ +getCsvIterator(); + $line = $file->current(); + if (!is_array($line)) { + return $data; + } + + $keyMap = $this->getKeyMap($line); + + $getValue = fn ($row, $key) => $row[$keyMap[$key] ?? null] ?? null; + + $file->next(); + while ($file->valid()) { + $row = $file->current(); + $file->next(); + if (empty($row)) { + continue; + } + + $name = $getValue($row, 'filename'); + if (empty($name)) { + continue; + } + + // Use null instead of ''. + $takenAt = $getValue($row, 'taken_at') ?: null; + + // If the file already exists but takenAt is null, replace the file by newly + // adding it. + /** @var VideoMetadata|null */ + $fileData = $data->getFile($name); + if (!is_null($fileData) && !is_null($takenAt)) { + $fileData->addFrame( + takenAt: $takenAt, + lat: $this->maybeCastToFloat($getValue($row, 'lat')), + lng: $this->maybeCastToFloat($getValue($row, 'lng')), + area: $this->maybeCastToFloat($getValue($row, 'area')), + distanceToGround: $this->maybeCastToFloat($getValue($row, 'distance_to_ground')), + gpsAltitude: $this->maybeCastToFloat($getValue($row, 'gps_altitude')), + yaw: $this->maybeCastToFloat($getValue($row, 'yaw')), + ); + } else { + $fileData = new VideoMetadata( + name: $getValue($row, 'filename'), + lat: $this->maybeCastToFloat($getValue($row, 'lat')), + lng: $this->maybeCastToFloat($getValue($row, 'lng')), + takenAt: $takenAt, + area: $this->maybeCastToFloat($getValue($row, 'area')), + distanceToGround: $this->maybeCastToFloat($getValue($row, 'distance_to_ground')), + gpsAltitude: $this->maybeCastToFloat($getValue($row, 'gps_altitude')), + yaw: $this->maybeCastToFloat($getValue($row, 'yaw')), + ); + + $data->addFile($fileData); + } + } + + return $data; + } +} diff --git a/app/Services/MetadataParsing/VideoMetadata.php b/app/Services/MetadataParsing/VideoMetadata.php new file mode 100644 index 000000000..d554301e2 --- /dev/null +++ b/app/Services/MetadataParsing/VideoMetadata.php @@ -0,0 +1,183 @@ +frames = collect([]); + + if (!is_null($takenAt)) { + $this->addFrame( + takenAt: $takenAt, + lat: $lat, + lng: $lng, + area: $area, + distanceToGround: $distanceToGround, + gpsAltitude: $gpsAltitude, + yaw: $yaw + ); + } + } + + public function getFrames(): Collection + { + return $this->frames; + } + + public function addFrame( + string $takenAt, + ?float $lat = null, + ?float $lng = null, + ?float $area = null, + ?float $distanceToGround = null, + ?float $gpsAltitude = null, + ?float $yaw = null + ): void { + $frame = new ImageMetadata( + name: $this->name, + takenAt: $takenAt, + lat: $lat, + lng: $lng, + area: $area, + distanceToGround: $distanceToGround, + gpsAltitude: $gpsAltitude, + yaw: $yaw + ); + $this->frames->push($frame); + } + + /** + * Determines if any metadata field other than the name is filled. + */ + public function isEmpty(): bool + { + return $this->frames->isEmpty() + && is_null($this->lat) + && is_null($this->lng) + && is_null($this->takenAt) + && is_null($this->area) + && is_null($this->distanceToGround) + && is_null($this->gpsAltitude) + && is_null($this->yaw); + } + + /** + * Get the array of metadata that can be used for Model::insert(); + */ + public function getInsertData(): array + { + if ($this->frames->isEmpty()) { + return $this->getInsertDataPlain(); + } + + return $this->getInsertDataFrames(); + + } + + /** + * Get the metadata insert array if no frames are present. + */ + protected function getInsertDataPlain(): array + { + $data = ['filename' => $this->name]; + + if (!is_null($this->lat)) { + $data['lat'] = [$this->lat]; + } + + if (!is_null($this->lng)) { + $data['lng'] = [$this->lng]; + } + + $attrs = []; + + if (!is_null($this->area)) { + $attrs['area'] = [$this->area]; + } + + if (!is_null($this->distanceToGround)) { + $attrs['distance_to_ground'] = [$this->distanceToGround]; + } + + if (!is_null($this->gpsAltitude)) { + $attrs['gps_altitude'] = [$this->gpsAltitude]; + } + + if (!is_null($this->yaw)) { + $attrs['yaw'] = [$this->yaw]; + } + + if (!empty($attrs)) { + $data['attrs'] = ['metadata' => $attrs]; + } + + return $data; + } + + /** + * Get the metadata insert array from all frames, sorted by taken_at. + * If one frame has data that another frame doesn't have, it is added as null. + */ + protected function getInsertDataFrames(): array + { + $data = [ + 'lat' => [], + 'lng' => [], + 'taken_at' => [], + ]; + + $attrs = [ + 'area' => [], + 'distance_to_ground' => [], + 'gps_altitude' => [], + 'yaw' => [], + ]; + + $timestamps = $this->frames + ->map(fn ($f) => Carbon::parse($f->takenAt)) + ->sort(fn ($a, $b) => $a->gt($b) ? 1 : -1); + + foreach ($timestamps as $index => $timestamp) { + $frame = $this->frames->get($index); + $data['lat'][] = $frame->lat; + $data['lng'][] = $frame->lng; + $data['taken_at'][] = $timestamp->toDateTimeString(); + + $attrs['area'][] = $frame->area; + $attrs['distance_to_ground'][] = $frame->distanceToGround; + $attrs['gps_altitude'][] = $frame->gpsAltitude; + $attrs['yaw'][] = $frame->yaw; + } + + // Remove all items that are full of null. + $data = array_filter($data, fn ($item) => !empty(array_filter($item, fn ($i) => !is_null($i)))); + + // Remove all items that are full of null. + $attrs = array_filter($attrs, fn ($item) => !empty(array_filter($item, fn ($i) => !is_null($i)))); + + $data['filename'] = $this->name; + + if (!empty($attrs)) { + $data['attrs'] = ['metadata' => $attrs]; + } + + return $data; + } +} diff --git a/app/Services/MetadataParsing/VolumeMetadata.php b/app/Services/MetadataParsing/VolumeMetadata.php new file mode 100644 index 000000000..f83a85488 --- /dev/null +++ b/app/Services/MetadataParsing/VolumeMetadata.php @@ -0,0 +1,202 @@ +files = collect([]); + } + + public function addFile(FileMetadata $file) + { + $this->files[$file->name] = $file; + } + + public function getFiles(): Collection + { + return $this->files->values(); + } + + public function getFile(string $name): ?FileMetadata + { + return $this->files->get($name); + } + + /** + * Determine if there is any file metadata. + */ + public function isEmpty(): bool + { + foreach ($this->files as $file) { + if (!$file->isEmpty()) { + return false; + } + } + + return true; + } + + public function hasAnnotations(): bool + { + foreach ($this->files as $file) { + if ($file->hasAnnotations()) { + return true; + } + } + + return false; + } + + public function hasFileLabels(): bool + { + foreach ($this->files as $file) { + if ($file->hasFileLabels()) { + return true; + } + } + + return false; + } + + /** + * The returned array is indexed by label IDs. + */ + public function getAnnotationLabels(array $onlyLabels = []): array + { + $labels = []; + + foreach ($this->files as $file) { + // Use union to automatically remove duplicates. + $labels += $file->getAnnotationLabels($onlyLabels); + } + + return $labels; + } + + /** + * The returned array is indexed by label IDs. + */ + public function getFileLabels(array $onlyLabels = []): array + { + $labels = []; + + foreach ($this->files as $file) { + // Use union to automatically remove duplicates. + $labels += $file->getFileLabelLabels($onlyLabels); + } + + return $labels; + } + + /** + * Get all users associated with annotations and/or file labels. + * + * @param array $onlyLabels List of metadata label IDs to filter the list of users. + * + * @return array Users indexed by ID. + */ + public function getUsers(array $onlyLabels = []): array + { + $users = []; + + foreach ($this->files as $file) { + // Use union to automatically remove duplicates. + $users += $file->getUsers($onlyLabels); + } + + return $users; + } + + /** + * @param array $map Optional map of metadata user IDs to database user IDs. Metadata + * users not in this list will be matched by their UUID (if any). + * @param array $onlyLabels Consider only users belonging to annotation labels or + * file labels with one of the specified metadata label IDs. + * + * @return array Map of metadata user IDs to database user IDs or null if no match + * was found. + */ + public function getMatchingUsers(array $map = [], array $onlyLabels = []): array + { + $users = $this->getUsers($onlyLabels); + + // Remove metadata user IDs that don't actually exist. + $idMap = array_flip(array_map(fn ($u) => $u->id, $users)); + $map = array_filter($map, fn ($id) => array_key_exists($id, $idMap), ARRAY_FILTER_USE_KEY); + + // Remove database user IDs that don't actually exist. + $idMap = DbUser::whereIn('id', array_unique($map))->pluck('id', 'id'); + $map = array_filter($map, fn ($id) => $idMap->has($id)); + + // Fetch database user IDs based on UUIDs. + $fetchUuids = array_filter( + $users, + fn ($u) => !array_key_exists($u->id, $map) && !is_null($u->uuid) + ); + $fetchUuids = array_map(fn ($u) => $u->uuid, $fetchUuids); + $uuidMap = DbUser::whereIn('uuid', $fetchUuids)->pluck('id', 'uuid'); + + foreach ($users as $user) { + if (array_key_exists($user->id, $map)) { + continue; + } + + $map[$user->id] = $uuidMap->get($user->uuid, null); + } + + return $map; + } + + /** + * @param array $map Optional map of metadata label IDs to database label IDs. + * Metadata labels not in this list will be matched by their UUID (if any). + * @param array $onlyLabels Consider only labels with one of the specified metadata + * label IDs. + * + * @return array Map of metadata label IDs to database label IDs or null if no match + * was found. + */ + public function getMatchingLabels(array $map = [], array $onlyLabels = []): array + { + $labels = $this->getAnnotationLabels($onlyLabels) + $this->getFileLabels($onlyLabels); + + // Remove metadata label IDs that don't actually exist. + $idMap = array_flip(array_map(fn ($l) => $l->id, $labels)); + $map = array_filter($map, fn ($id) => array_key_exists($id, $idMap), ARRAY_FILTER_USE_KEY); + + // Remove database label IDs that don't actually exist. + $idMap = DbLabel::whereIn('id', array_unique($map))->pluck('id', 'id'); + $map = array_filter($map, fn ($id) => $idMap->has($id)); + + // Fetch database label IDs based on UUIDs. + $fetchUuids = array_filter( + $labels, + fn ($l) => !array_key_exists($l->id, $map) && !is_null($l->uuid) + ); + $fetchUuids = array_map(fn ($l) => $l->uuid, $fetchUuids); + $uuidMap = DbLabel::whereIn('uuid', $fetchUuids)->pluck('id', 'uuid'); + + foreach ($labels as $label) { + if (array_key_exists($label->id, $map)) { + continue; + } + + $map[$label->id] = $uuidMap->get($label->uuid, null); + } + + return $map; + } +} diff --git a/app/Traits/ChecksMetadataStrings.php b/app/Traits/ChecksMetadataStrings.php deleted file mode 100644 index ff965a8c2..000000000 --- a/app/Traits/ChecksMetadataStrings.php +++ /dev/null @@ -1,14 +0,0 @@ -metadata_file_path); + } + + public function saveMetadata(UploadedFile $file): void + { + $this->metadata_file_path = "$this->id"; + if ($extension = $file->getClientOriginalExtension()) { + $this->metadata_file_path .= '.'.$extension; + } + $file->storeAs('', $this->metadata_file_path, static::$metadataFileDisk); + $this->save(); + } + + public function getMetadata(): ?VolumeMetadata + { + if (!$this->hasMetadata()) { + return null; + } + + $disk = static::$metadataFileDisk; + $key = "metadata-{$disk}-{$this->metadata_file_path}"; + + return Cache::store('array')->remember($key, 60, function () use ($disk) { + $tmpPath = tempnam(sys_get_temp_dir(), 'metadata'); + try { + $to = fopen($tmpPath, 'w'); + $from = Storage::disk($disk)->readStream($this->metadata_file_path); + stream_copy_to_stream($from, $to); + $type = ($this->media_type_id === MediaType::imageId()) ? 'image' : 'video'; + + $parser = new $this->metadata_parser(new SplFileInfo($tmpPath)); + + return $parser->getMetadata(); + } finally { + if (isset($to) && is_resource($to)) { + fclose($to); + } + File::delete($tmpPath); + } + }); + } + + /** + * @param boolean $noUpdate Do not set metadata_file_path to null. + */ + public function deleteMetadata($noUpdate = false): void + { + if ($this->hasMetadata()) { + Storage::disk(static::$metadataFileDisk)->delete($this->metadata_file_path); + if (!$noUpdate) { + $this->update(['metadata_file_path' => null]); + } + } + } +} diff --git a/app/Traits/ParsesMetadata.php b/app/Traits/ParsesMetadata.php deleted file mode 100644 index 755699399..000000000 --- a/app/Traits/ParsesMetadata.php +++ /dev/null @@ -1,285 +0,0 @@ - 'filename', - 'lon' => 'lng', - 'longitude' => 'lng', - 'latitude' => 'lat', - 'heading' => 'yaw', - 'sub_datetime' => 'taken_at', - 'sub_longitude' => 'lng', - 'sub_latitude' => 'lat', - 'sub_heading' => 'yaw', - 'sub_distance' => 'distance_to_ground', - 'sub_altitude' => 'gps_altitude', - - ]; - - /** - * Maps iFDO field names to BIIGLE metadata CSV fields. - * - * @var array - */ - protected $ifdoFieldMap = [ - 'image-area-square-meter' => 'area', - 'image-meters-above-ground' => 'distance_to_ground', - 'image-altitude' => 'gps_altitude', - 'image-latitude' => 'lat', - 'image-longitude' => 'lng', - 'image-datetime' => 'taken_at', - 'image-camera-yaw-degrees' => 'yaw', - ]; - - /** - * Parse a metadata CSV string to an array. - */ - public function parseMetadata(string $content): array - { - // Split string by rows but respect possible escaped linebreaks. - $rows = str_getcsv($content, "\n"); - // Now parse individual rows. - $rows = array_map('str_getcsv', $rows); - - $rows[0][0] = Util::removeBom($rows[0][0]); - - $rows[0] = array_map('strtolower', $rows[0]); - - $rows[0] = array_map(function ($column) { - if (array_key_exists($column, $this->columnSynonyms)) { - return $this->columnSynonyms[$column]; - } - - return $column; - }, $rows[0]); - - return $rows; - } - - /** - * Parse metadata from a CSV file to an array. - * - * @param UploadedFile $file - * - * @return array - */ - public function parseMetadataFile(UploadedFile $file) - { - return $this->parseMetadata($file->get()); - } - - /** - * Parse a volume metadata iFDO YAML string to an array. - * - * See: https://marine-imaging.com/fair/ifdos/iFDO-overview/ - * - * @param string $content - * - * @return array - */ - public function parseIfdo($content) - { - try { - $yaml = yaml_parse($content); - } catch (Exception $e) { - throw new Exception("The YAML file could not be parsed."); - } - - if (!is_array($yaml)) { - throw new Exception("The file does not seem to be a valid iFDO."); - } - - if (!array_key_exists('image-set-header', $yaml)) { - throw new Exception("The 'image-set-header' key must be present."); - } - - $header = $yaml['image-set-header']; - - if (!array_key_exists('image-set-name', $header)) { - throw new Exception("The 'image-set-name' key must be present."); - } - - if (!array_key_exists('image-set-handle', $header)) { - throw new Exception("The 'image-set-handle' key must be present."); - } - - if (!$this->isValidHandle($header['image-set-handle'])) { - throw new Exception("The 'image-set-handle' key must be a valid handle."); - } - - if (!array_key_exists('image-set-uuid', $header)) { - throw new Exception("The 'image-set-uuid' key must be present."); - } - - $url = ''; - if (array_key_exists('image-set-data-handle', $header)) { - if (!$this->isValidHandle($header['image-set-data-handle'])) { - throw new Exception("The 'image-set-data-handle' key must be a valid handle."); - } - - $url = 'https://hdl.handle.net/'.$header['image-set-data-handle']; - } - - $mediaType = 'image'; - - if (array_key_exists('image-acquisition', $header) && $header['image-acquisition'] === 'video') { - $mediaType = 'video'; - } - - $files = []; - if (array_key_exists('image-set-items', $yaml)) { - $files = $this->parseIfdoItems($header, $yaml['image-set-items']); - } - - return [ - 'name' => $header['image-set-name'], - 'handle' => $header['image-set-handle'], - 'uuid' => $header['image-set-uuid'], - 'url' => $url, - 'media_type' => $mediaType, - 'files' => $files, - ]; - } - - /** - * Parse a volume metadata iFDO YAML file to an array. - * - * @param UploadedFile $file - * - * @return array - */ - public function parseIfdoFile(UploadedFile $file) - { - return $this->parseIfdo($file->get()); - } - - /** - * Parse iFDO image-set-items to a CSV-like metadata array that can be parsed by - * `parseMetadata` if converted to a string. - * - * @param array $header iFDO image-set-header - * @param array $items iFDO image-set-items. Passed by reference so potentially huge arrays are not copied. - * - * @return array - */ - protected function parseIfdoItems($header, &$items) - { - $fields = []; - $rows = []; - $reverseFieldMap = array_flip($this->ifdoFieldMap); - - if (array_key_exists('image-depth', $header)) { - $header['image-altitude'] = -1 * $header['image-depth']; - } - - $leftToCheck = $this->ifdoFieldMap; - - // Add all metadata fields present in header. - foreach ($leftToCheck as $ifdoField => $csvField) { - if (array_key_exists($ifdoField, $header)) { - $fields[] = $csvField; - unset($leftToCheck[$ifdoField]); - } - } - - // Normalize image-set-items entries. An entry can be either a list (e.g. for a - // video) or an object (e.g. for an image). But an image could be a list with a - // single entry, too. - foreach ($items as &$item) { - if (!is_array($item)) { - $item = [null]; - } elseif (!array_key_exists(0, $item)) { - $item = [$item]; - } - } - - // Convert item depth to altitude. - // Also add all metadata fields present in items (stop early). - foreach ($items as &$subItems) { - foreach ($subItems as &$subItem) { - if (empty($subItem)) { - continue; - } - - if (array_key_exists('image-depth', $subItem)) { - $subItem['image-altitude'] = -1 * $subItem['image-depth']; - // Save some memory for potentially huge arrays. - unset($subItem['image-depth']); - } - - foreach ($leftToCheck as $ifdoField => $csvField) { - if (array_key_exists($ifdoField, $subItem)) { - $fields[] = $csvField; - unset($leftToCheck[$ifdoField]); - if (empty($leftToCheck)) { - break; - } - } - } - } - unset($subItem); // Important to destroy by-reference variable after the loop! - } - unset($subItems); // Important to destroy by-reference variable after the loop! - - sort($fields); - - foreach ($items as $filename => $subItems) { - $defaults = []; - foreach ($subItems as $index => $subItem) { - if ($index === 0 && is_array($subItem)) { - $defaults = $subItem; - } - - $row = [$filename]; - foreach ($fields as $field) { - $ifdoField = $reverseFieldMap[$field]; - if (is_array($subItem) && array_key_exists($ifdoField, $subItem)) { - // Take field value of subItem if it is given. - $row[] = $subItem[$ifdoField]; - } elseif (array_key_exists($ifdoField, $defaults)) { - // Otherwise fall back to the defaults of the first subItem. - $row[] = $defaults[$ifdoField]; - } elseif (array_key_exists($ifdoField, $header)) { - // Otherwise fall back to the defaults of the header. - $row[] = $header[$ifdoField]; - } else { - $row[] = ''; - } - } - - $rows[] = $row; - } - } - - // Add this only not because it should not be included in sort earlier and it is - // should be skipped in the loop above. - array_unshift($fields, 'filename'); - array_unshift($rows, $fields); - - return $rows; - } - - /** - * Determine if a value is a valid handle. - * - * @param string $value - * - * @return boolean - */ - protected function isValidHandle($value) - { - return preg_match('/[^\/]+\/[^\/]/', $value); - } -} diff --git a/app/Video.php b/app/Video.php index 6087299db..eb9a904c1 100644 --- a/app/Video.php +++ b/app/Video.php @@ -79,6 +79,9 @@ class Video extends VolumeFile 'uuid', 'attrs', 'duration', + 'lng', + 'lat', + 'taken_at', ]; /** @@ -200,11 +203,15 @@ public function labels() * * @param array $value */ - public function setTakenAtAttribute(array $value) + public function setTakenAtAttribute(?array $value) { - $value = array_map([Carbon::class, 'parse'], $value); + if (is_array($value)) { + $value = array_map([Carbon::class, 'parse'], $value); - $this->attributes['taken_at'] = json_encode($value); + $this->attributes['taken_at'] = json_encode($value); + } else { + $this->attributes['taken_at'] = $value; + } } /** diff --git a/app/VideoAnnotation.php b/app/VideoAnnotation.php index cce6798cf..8739e1972 100644 --- a/app/VideoAnnotation.php +++ b/app/VideoAnnotation.php @@ -102,7 +102,11 @@ public function validatePoints(array $points = []) } // Gaps are represented as empty arrays - array_map(function ($point) { if (count($point)) { parent::validatePoints($point); } }, $this->points); + array_map(function ($point) { + if (count($point)) { + parent::validatePoints($point); + } + }, $this->points); } /** diff --git a/app/Volume.php b/app/Volume.php index e3907a728..376608b9f 100644 --- a/app/Volume.php +++ b/app/Volume.php @@ -3,15 +3,12 @@ namespace Biigle; use Biigle\Traits\HasJsonAttributes; +use Biigle\Traits\HasMetadataFile; use Cache; use Carbon\Carbon; use DB; -use Exception; use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Database\Eloquent\Model; -use Illuminate\Http\Response; -use Illuminate\Http\UploadedFile; -use Illuminate\Support\Facades\Storage; /** * A volume is a collection of images. Volumes belong to one or many @@ -19,7 +16,7 @@ */ class Volume extends Model { - use HasJsonAttributes, HasFactory; + use HasJsonAttributes, HasFactory, HasMetadataFile; /** * Regular expression that matches the supported image file extensions. @@ -41,6 +38,21 @@ class Volume extends Model */ const VIDEO_FILE_REGEX = '/\.(mpe?g|mp4|webm)(\?.+)?$/i'; + /** + * The attributes that are mass assignable. + * + * @var array + */ + protected $fillable = [ + 'name', + 'url', + 'media_type_id', + 'handle', + 'creator_id', + 'metadata_file_path', + 'metadata_parser', + ]; + /** * The attributes hidden from the model's JSON form. * @@ -61,6 +73,11 @@ class Volume extends Model 'media_type_id' => 'int', ]; + protected static function booted(): void + { + static::$metadataFileDisk = config('volumes.metadata_storage_disk'); + } + /** * Parses a comma separated list of filenames to an array. * @@ -441,85 +458,6 @@ public function isVideoVolume() return $this->media_type_id === MediaType::videoId(); } - /** - * Save an iFDO metadata file and link it with this volume. - * - * @param UploadedFile $file iFDO YAML file. - * - */ - public function saveIfdo(UploadedFile $file) - { - $disk = config('volumes.ifdo_storage_disk'); - $file->storeAs('', $this->getIfdoFilename(), $disk); - Cache::forget($this->getIfdoCacheKey()); - } - - /** - * Check if an iFDO metadata file is available for this volume. - * - * @param bool $ignoreErrors Set to `true` to ignore exceptions and return `false` if iFDO existence could not be determined. - * @return boolean - */ - public function hasIfdo($ignoreErrors = false) - { - try { - return Cache::remember($this->getIfdoCacheKey(), 3600, fn () => Storage::disk(config('volumes.ifdo_storage_disk'))->exists($this->getIfdoFilename())); - } catch (Exception $e) { - if (!$ignoreErrors) { - throw $e; - } - - return false; - } - } - - /** - * Delete the iFDO metadata file linked with this volume. - */ - public function deleteIfdo() - { - Storage::disk(config('volumes.ifdo_storage_disk'))->delete($this->getIfdoFilename()); - Cache::forget($this->getIfdoCacheKey()); - } - - /** - * Download the iFDO that is attached to this volume. - * - * @return \Symfony\Component\HttpFoundation\StreamedResponse - */ - public function downloadIfdo() - { - $disk = Storage::disk(config('volumes.ifdo_storage_disk')); - - if (!$disk->exists($this->getIfdoFilename())) { - abort(Response::HTTP_NOT_FOUND); - } - - return $disk->download($this->getIfdoFilename(), "biigle-volume-{$this->id}-ifdo.yaml"); - } - - /** - * Get the content of the iFDO file associated with this volume. - * - * @return array - */ - public function getIfdo() - { - $content = Storage::disk(config('volumes.ifdo_storage_disk'))->get($this->getIfdoFilename()); - - return yaml_parse($content); - } - - /** - * Get the filename of the volume iFDO in storage. - * - * @return string - */ - protected function getIfdoFilename() - { - return $this->id.'.yaml'; - } - /** * Get the cache key for volume thumbnails. * @@ -539,14 +477,4 @@ protected function getGeoInfoCacheKey() { return "volume-{$this->id}-has-geo-info"; } - - /** - * Get the cache key for volume iFDO info. - * - * @return string - */ - protected function getIfdoCacheKey() - { - return "volume-{$this->id}-has-ifdo"; - } } diff --git a/app/VolumeFile.php b/app/VolumeFile.php index a7f8238c3..af7771d6c 100644 --- a/app/VolumeFile.php +++ b/app/VolumeFile.php @@ -54,20 +54,16 @@ public function volume() /** * Set the metadata attribute. - * - * @param array $value */ - public function setMetadataAttribute(array $value) + public function setMetadataAttribute(?array $value) { return $this->setJsonAttr('metadata', $value); } /** * Get the metadata attribute. - * - * @return array */ - public function getMetadataAttribute() + public function getMetadataAttribute(): ?array { return $this->getJsonAttr('metadata', []); } diff --git a/config/filesystems.php b/config/filesystems.php index d5e3f0196..f35755d47 100644 --- a/config/filesystems.php +++ b/config/filesystems.php @@ -62,9 +62,14 @@ 'visibility' => 'public', ], - 'ifdos' => [ + 'metadata' => [ 'driver' => 'local', - 'root' => storage_path('ifdos'), + 'root' => storage_path('metadata'), + ], + + 'pending-metadata' => [ + 'driver' => 'local', + 'root' => storage_path('pending-metadata'), ], ], diff --git a/config/volumes.php b/config/volumes.php index e413a61ee..98723f697 100644 --- a/config/volumes.php +++ b/config/volumes.php @@ -17,7 +17,13 @@ 'editor_storage_disks' => array_filter(explode(',', env('VOLUME_EDITOR_STORAGE_DISKS', ''))), /* - | Storage disk for iFDO metadata files linked with volumes. + | Storage disk for metadata files linked with volumes. */ - 'ifdo_storage_disk' => env('VOLUME_IFDO_STORAGE_DISK', 'ifdos'), + 'metadata_storage_disk' => env('VOLUME_METADATA_STORAGE_DISK', 'metadata'), + + + /* + | Storage disk for metadata files of pending volumes. + */ + 'pending_metadata_storage_disk' => env('VOLUME_PENDING_METADATA_STORAGE_DISK', 'pending-metadata'), ]; diff --git a/database/factories/PendingVolumeFactory.php b/database/factories/PendingVolumeFactory.php new file mode 100644 index 000000000..320c82565 --- /dev/null +++ b/database/factories/PendingVolumeFactory.php @@ -0,0 +1,38 @@ + + */ +class PendingVolumeFactory extends Factory +{ + /** + * The name of the factory's corresponding model. + * + * @var class-string + */ + protected $model = PendingVolume::class; + + /** + * Define the model's default state. + * + * @return array + */ + public function definition(): array + { + return [ + 'media_type_id' => fn () => MediaType::imageId(), + 'user_id' => User::factory(), + 'project_id' => Project::factory(), + ]; + } +} diff --git a/database/migrations/2024_03_06_143800_create_pending_volumes_table.php b/database/migrations/2024_03_06_143800_create_pending_volumes_table.php new file mode 100644 index 000000000..4286c5d16 --- /dev/null +++ b/database/migrations/2024_03_06_143800_create_pending_volumes_table.php @@ -0,0 +1,84 @@ +id(); + $table->timestamps(); + + $table->foreignId('user_id') + ->constrained() + ->onDelete('cascade'); + + $table->foreignId('media_type_id') + ->constrained() + ->onDelete('restrict'); + + $table->foreignId('project_id') + ->constrained() + ->onDelete('cascade'); + + // This is used if annotations or file labels should be imported. The volume + // will be created first but the pending volume is still required to store + // additional information required for the import. + $table->foreignId('volume_id') + ->nullable() + ->constrained() + ->onDelete('cascade'); + + // Specify if the pending volume should be used to import annotations. + $table->boolean('import_annotations')->default(false); + + // Specify if the pending volume should be used to import file labels. + $table->boolean('import_file_labels')->default(false); + + // Specifies if a job to import metadata is already dispatched for this + // pending volume. + $table->boolean('importing')->default(false); + + // Path of the file in the pending_metadata_storage_disk. + $table->string('metadata_file_path', 256)->nullable(); + // Class name of the metadata parser for the metadata file. + $table->string('metadata_parser', 256)->nullable(); + + // Used to filter the imported annotations. + $table->jsonb('only_annotation_labels')->nullable(); + // Used to filter the imported file labels. + $table->jsonb('only_file_labels')->nullable(); + // Used to map labels from the metadata to labels in the database. + $table->jsonb('label_map')->nullable(); + // Used to map users from the metadata to users in the database. + $table->jsonb('user_map')->nullable(); + + // A user is only allowed to create one pending volume at a time for a + // project. + $table->unique(['user_id', 'project_id']); + }); + + Schema::table('volumes', function (Blueprint $table) { + $table->string('metadata_file_path', 256)->nullable(); + $table->string('metadata_parser', 256)->nullable(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('volumes', function (Blueprint $table) { + $table->dropColumn('metadata_file_path'); + $table->dropColumn('metadata_parser'); + }); + + Schema::dropIfExists('pending_volumes'); + } +}; diff --git a/public/assets/images/ifdo_logo_grey.svg b/public/assets/images/ifdo_logo_grey.svg deleted file mode 100644 index c0413811b..000000000 --- a/public/assets/images/ifdo_logo_grey.svg +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - diff --git a/resources/assets/js/core/components/typeahead.vue b/resources/assets/js/core/components/typeahead.vue index 9870607dd..7f514f350 100644 --- a/resources/assets/js/core/components/typeahead.vue +++ b/resources/assets/js/core/components/typeahead.vue @@ -19,7 +19,8 @@ item-key="name" > @@ -46,7 +47,6 @@ import TypeaheadItem from './typeaheadItem'; export default { components: { typeahead: Typeahead, - typeaheadItem: TypeaheadItem, }, props: { items: { @@ -77,6 +77,10 @@ export default { type: Number, default: 5, }, + itemComponent: { + type: Object, + default: () => TypeaheadItem, + }, }, data() { return { diff --git a/resources/assets/js/label-trees/components/labelTypeahead.vue b/resources/assets/js/label-trees/components/labelTypeahead.vue index d7d6198e1..fc31923a4 100644 --- a/resources/assets/js/label-trees/components/labelTypeahead.vue +++ b/resources/assets/js/label-trees/components/labelTypeahead.vue @@ -1,4 +1,5 @@ diff --git a/resources/assets/js/volumes/api/parseIfdoFile.js b/resources/assets/js/volumes/api/parseIfdoFile.js deleted file mode 100644 index 388ca7e93..000000000 --- a/resources/assets/js/volumes/api/parseIfdoFile.js +++ /dev/null @@ -1,10 +0,0 @@ -/** - * Resource for uploading an IFDO file and getting content as JSON. - * - * let resource = parseIfdoFile; - * let data = new FormData(); - * data.append('file', fileInputElement.files[0]); - * - * resource.save(data).then(...); - */ -export default Vue.resource('api/v1/volumes/parse-ifdo'); diff --git a/resources/assets/js/volumes/api/volumeIfdo.js b/resources/assets/js/volumes/api/volumeIfdo.js deleted file mode 100644 index 6cc046575..000000000 --- a/resources/assets/js/volumes/api/volumeIfdo.js +++ /dev/null @@ -1,8 +0,0 @@ -/** - * Resource for getting and deleting iFDO files attached to a volume. - * - * let resource = biigle.$require('api.volumeIfdo'); - * - * resource.delete({id: volumeId}).then(...); - */ -export default Vue.resource('api/v1/volumes{/id}/ifdo'); diff --git a/resources/assets/js/volumes/components/labelMapping.vue b/resources/assets/js/volumes/components/labelMapping.vue new file mode 100644 index 000000000..f1d4fd851 --- /dev/null +++ b/resources/assets/js/volumes/components/labelMapping.vue @@ -0,0 +1,50 @@ + + + diff --git a/resources/assets/js/volumes/components/labelMappingItem.vue b/resources/assets/js/volumes/components/labelMappingItem.vue new file mode 100644 index 000000000..b7a13e405 --- /dev/null +++ b/resources/assets/js/volumes/components/labelMappingItem.vue @@ -0,0 +1,204 @@ + + + diff --git a/resources/assets/js/volumes/components/userMapping.vue b/resources/assets/js/volumes/components/userMapping.vue new file mode 100644 index 000000000..e9c4b06e9 --- /dev/null +++ b/resources/assets/js/volumes/components/userMapping.vue @@ -0,0 +1,45 @@ + + + diff --git a/resources/assets/js/volumes/components/userMappingItem.vue b/resources/assets/js/volumes/components/userMappingItem.vue new file mode 100644 index 000000000..f0f1e6c4d --- /dev/null +++ b/resources/assets/js/volumes/components/userMappingItem.vue @@ -0,0 +1,89 @@ + + + diff --git a/resources/assets/js/volumes/createFormStep1.vue b/resources/assets/js/volumes/createFormStep1.vue new file mode 100644 index 000000000..3107c9510 --- /dev/null +++ b/resources/assets/js/volumes/createFormStep1.vue @@ -0,0 +1,77 @@ + diff --git a/resources/assets/js/volumes/createForm.vue b/resources/assets/js/volumes/createFormStep2.vue similarity index 76% rename from resources/assets/js/volumes/createForm.vue rename to resources/assets/js/volumes/createFormStep2.vue index 8f51f06fb..15a9f5367 100644 --- a/resources/assets/js/volumes/createForm.vue +++ b/resources/assets/js/volumes/createFormStep2.vue @@ -3,13 +3,8 @@ import BrowserApi from './api/browser'; import Dropdown from 'uiv/dist/Dropdown'; import FileBrowser from '../core/components/fileBrowser'; import LoaderMixin from '../core/mixins/loader'; -import ParseIfdoFileApi from '../volumes/api/parseIfdoFile'; -import {handleErrorResponse} from '../core/messages/store'; - -const MEDIA_TYPE = { - IMAGE: 'image', - VIDEO: 'video', -}; +import {debounce} from '../core/utils'; +import {MEDIA_TYPE} from './createFormStep1'; const FILE_SOURCE = { REMOTE: 'remote', @@ -19,9 +14,6 @@ const FILE_SOURCE = { const numberFormatter = new Intl.NumberFormat(); -/** - * View model for the create volume form. - */ export default { mixins: [LoaderMixin], components: { @@ -32,17 +24,20 @@ export default { return { disks: [], filenames: '', - filenamesReadFromMetadata: false, + filesDontMatchMetadata: false, fileSource: FILE_SOURCE.REMOTE, - hadMetadataText: false, handle: '', imageDiskCache: {}, + importAnnotations: false, + importFileLabels: false, + initialized: false, initializingBrowser: false, loadingBrowser: false, - loadingImport: false, mediaType: MEDIA_TYPE.IMAGE, - metadataText: '', + metadataFilenames: [], name: '', + remoteFilenames: '', + remoteUrl: '', selectedDiskRoot: null, storageDisk: null, url: '', @@ -65,12 +60,6 @@ export default { isRemoteImageVolume() { return this.isImageMediaType && this.url.search(/^https?:\/\//) !== -1; }, - hasMetadata() { - return this.metadataText.length > 0; - }, - showImportAgainMessage() { - return this.hadMetadataText && !this.hasMetadata; - }, isRemoteFileSource() { return this.fileSource === FILE_SOURCE.REMOTE; }, @@ -108,6 +97,36 @@ export default { fileCountText() { return numberFormatter.format(this.fileCount); }, + remoteButtonClass() { + return { + active: this.isRemoteFileSource, + 'btn-info': this.isRemoteFileSource, + }; + }, + userDiskButtonClass() { + return { + active: this.isUserDiskFileSource, + 'btn-info': this.isUserDiskFileSource, + }; + }, + diskButtonClass() { + return { + active: this.isDiskFileSource, + 'btn-info': this.isDiskFileSource, + }; + }, + importAnnotationsButtonClass() { + return { + active: this.importAnnotations, + 'btn-info': this.importAnnotations, + }; + }, + importFileLabelsButtonClass() { + return { + active: this.importFileLabels, + 'btn-info': this.importFileLabels, + }; + }, }, methods: { fetchDirectories(disk, path) { @@ -164,90 +183,23 @@ export default { this.mediaType = MEDIA_TYPE.VIDEO; this.resetFileSource(); }, - setCsvMetadata(event) { - this.hasMetadataCsv = true; - let file = event.target.files[0]; - this.readCsvMetadataText(file).then((text) => { - this.metadataText = text; - if (!this.filenames && this.isRemoteFileSource) { - this.filenames = this.parseMetadataTextFilenames(text); - this.filenamesReadFromMetadata = true; - } - // Reset input field so the file is not uploaded, too. - event.target.value = ''; - }) - }, - readCsvMetadataText(file) { - let reader = new FileReader(); - let promise = new Promise(function (resolve, reject) { - reader.onload = resolve; - reader.onerror = reject; - }); - reader.readAsText(file); - - return promise.then(function () { - return reader.result; - }); - }, - parseMetadataTextFilenames(text) { - let metadata = text.split("\n").map(row => row.split(',')); - - return this.parseMetadataFilenames(metadata); - }, - importCsv() { - this.$refs.metadataCsvField.click(); - }, - importIfdo() { - this.$refs.metadataIfdoField.click(); - }, - parseIfdoMetadata(event) { - let data = new FormData(); - data.append('file', event.target.files[0]); - this.loadingImport = true; - ParseIfdoFileApi.save(data) - .then(this.setIfdoMetadata, handleErrorResponse) - .finally(() => this.loadingImport = false); - }, - setIfdoMetadata(response) { - let ifdo = response.body; - if (!this.name) { - this.name = ifdo.name; - } - if (!this.url && this.isRemoteFileSource) { - this.url = ifdo.url; - } - if (!this.handle) { - this.handle = ifdo.handle; - } - if (!this.filenames && this.isRemoteFileSource) { - this.filenames = this.parseMetadataFilenames(ifdo.files); - this.filenamesReadFromMetadata = true; - } - - this.metadataText = ifdo.files.map(row => row.join(',')).join("\n"); - }, - parseMetadataFilenames(metadata) { - let columns = metadata[0]; - let filenameColumn = columns.indexOf('filename') - let filenames = metadata.slice(1).map(row => row[filenameColumn]); - - // Remove duplicate filenames (possible for video metadata). - return Array.from(new Set(filenames)).join(', '); - }, - clearMetadata() { - this.metadataText = ''; - this.$refs.metadataIfdoField.value = ''; - }, selectRemoteFileSource() { if (!this.isRemoteFileSource) { this.fileSource = FILE_SOURCE.REMOTE; this.storageDisk = null; this.selectedDiskRoot = null; - this.url = ''; - this.filenames = ''; + this.url = this.remoteUrl; + this.filenames = this.remoteFilenames; } }, selectStorageDisk(disk) { + if (!this.storageDisk) { + // Make a backup so the remote filenames and URL can be restored if the + // user switches back from a storage disk to a remote source. + this.remoteFilenames = this.filenames; + this.remoteUrl = this.url; + } + if (this.storageDisk !== disk) { if (this.disks.includes(disk)) { this.fileSource = FILE_SOURCE.DISK; @@ -268,7 +220,7 @@ export default { if (!cache.hasOwnProperty(disk)) { cache[disk] = this.fetchDirectoryContent(disk, '') - .then(this.setStorageDiskRoot, handleErrorResponse); + .then(this.setStorageDiskRoot, this.handleErrorResponse); } return cache[disk]; @@ -296,7 +248,7 @@ export default { directory.loaded = true; return directory; - }, handleErrorResponse) + }, this.handleErrorResponse) .finally(() => directory.loading = false); }, unselectAllDirectories(directory) { @@ -326,7 +278,7 @@ export default { setUrlAndFilenames(path, files) { // Add only one slash, as path already has a leading slash. this.url = `${this.storageDisk}:/${path}`; - this.filenames = files.map(file => file.name).join(', '); + this.filenames = files.map(file => file.name).join(','); }, unselectDirectory(directory) { this.unselectAllDirectories(directory); @@ -415,6 +367,12 @@ export default { this.selectFile(file, directory, path, event); } }, + toggleImportAnnotations() { + this.importAnnotations = !this.importAnnotations; + }, + toggleImportFileLabels() { + this.importFileLabels = !this.importFileLabels; + }, }, watch: { storageDisk(disk) { @@ -430,11 +388,29 @@ export default { this.unselectAllDirectories(oldRoot); } }, - hasMetadata(hasMetadata) { - if (hasMetadata) { - // Don't show message again once a metadata file had been selected. - this.hadMetadataText = false; + filenames() { + if (this.metadataFilenames.length === 0 || !this.filenames || !this.filenames.includes('.')) { + this.filesDontMatchMetadata = false; + return; } + + // Use a watcher+debounce instead of a computed property because this may be + // called on each keystroke in the textarea. + debounce(() => { + if (!this.filenames || !this.filenames.includes('.')) { + this.filesDontMatchMetadata = false; + return; + } + + for (var i = this.metadataFilenames.length - 1; i >= 0; i--) { + if (this.filenames.includes(this.metadataFilenames[i])) { + this.filesDontMatchMetadata = false; + return; + } + } + + this.filesDontMatchMetadata = true; + }, 1000, 'compare-volume-filenames-with-metadata'); }, }, created() { @@ -442,9 +418,11 @@ export default { this.url = biigle.$require('volumes.url'); this.name = biigle.$require('volumes.name'); this.handle = biigle.$require('volumes.handle'); - this.hadMetadataText = biigle.$require('volumes.hadMetadataText'); this.mediaType = biigle.$require('volumes.mediaType'); this.filenames = biigle.$require('volumes.filenames'); + if (biigle.$require('volumes.filenamesFromMeta')) { + this.metadataFilenames = this.filenames.split(','); + } let [disk, path] = this.url.split('://'); if (this.disks.includes(disk)) { @@ -454,6 +432,8 @@ export default { mounted() { // Vue disables the autofocus attribute somehow, so set focus manually here. this.$refs.nameInput.focus(); + // Used to mask some flashing elements on pageload. + this.initialized = true; }, }; diff --git a/resources/assets/js/volumes/createFormStep3.vue b/resources/assets/js/volumes/createFormStep3.vue new file mode 100644 index 000000000..9f507cb30 --- /dev/null +++ b/resources/assets/js/volumes/createFormStep3.vue @@ -0,0 +1,45 @@ + diff --git a/resources/assets/js/volumes/createFormStep4.vue b/resources/assets/js/volumes/createFormStep4.vue new file mode 100644 index 000000000..80a3f9c04 --- /dev/null +++ b/resources/assets/js/volumes/createFormStep4.vue @@ -0,0 +1,45 @@ + diff --git a/resources/assets/js/volumes/createFormStep5.vue b/resources/assets/js/volumes/createFormStep5.vue new file mode 100644 index 000000000..3a5f378c7 --- /dev/null +++ b/resources/assets/js/volumes/createFormStep5.vue @@ -0,0 +1,79 @@ + diff --git a/resources/assets/js/volumes/createFormStep6.vue b/resources/assets/js/volumes/createFormStep6.vue new file mode 100644 index 000000000..a5401bde9 --- /dev/null +++ b/resources/assets/js/volumes/createFormStep6.vue @@ -0,0 +1,67 @@ + diff --git a/resources/assets/js/volumes/createFormStep7.vue b/resources/assets/js/volumes/createFormStep7.vue new file mode 100644 index 000000000..742b5bb00 --- /dev/null +++ b/resources/assets/js/volumes/createFormStep7.vue @@ -0,0 +1,7 @@ + diff --git a/resources/assets/js/volumes/main.js b/resources/assets/js/volumes/main.js index 95e8cd011..396be858c 100644 --- a/resources/assets/js/volumes/main.js +++ b/resources/assets/js/volumes/main.js @@ -1,6 +1,12 @@ import './export'; import AnnotationSessionPanel from './annotationSessionPanel'; -import CreateForm from './createForm'; +import CreateFormStep1 from './createFormStep1'; +import CreateFormStep2 from './createFormStep2'; +import CreateFormStep3 from './createFormStep3'; +import CreateFormStep4 from './createFormStep4'; +import CreateFormStep5 from './createFormStep5'; +import CreateFormStep6 from './createFormStep6'; +import CreateFormStep7 from './createFormStep7'; import CloneForm from './cloneForm'; import FileCount from './fileCount'; import FilePanel from './filePanel'; @@ -10,7 +16,13 @@ import SearchResults from './searchResults'; import VolumeContainer from './volumeContainer'; biigle.$mount('annotation-session-panel', AnnotationSessionPanel); -biigle.$mount('create-volume-form', CreateForm); +biigle.$mount('create-volume-form-step-1', CreateFormStep1); +biigle.$mount('create-volume-form-step-2', CreateFormStep2); +biigle.$mount('create-volume-form-step-3', CreateFormStep3); +biigle.$mount('create-volume-form-step-4', CreateFormStep4); +biigle.$mount('create-volume-form-step-5', CreateFormStep5); +biigle.$mount('create-volume-form-step-6', CreateFormStep6); +biigle.$mount('create-volume-form-step-7', CreateFormStep7); biigle.$mount('clone-volume-form', CloneForm); biigle.$mount('file-panel', FilePanel); biigle.$mount('projects-breadcrumb', ProjectsBreadcrumb); diff --git a/resources/assets/js/volumes/metadataUpload.vue b/resources/assets/js/volumes/metadataUpload.vue index 2d4a72c90..15d183354 100644 --- a/resources/assets/js/volumes/metadataUpload.vue +++ b/resources/assets/js/volumes/metadataUpload.vue @@ -2,10 +2,6 @@ import Dropdown from 'uiv/dist/Dropdown'; import LoaderMixin from '../core/mixins/loader'; import MetadataApi from './api/volumeMetadata'; -import ParseIfdoFileApi from './api/parseIfdoFile'; -import VolumeIfdoApi from './api/volumeIfdo'; -import Tab from 'uiv/dist/Tab'; -import Tabs from 'uiv/dist/Tabs'; import MessageStore from '../core/messages/store'; /** @@ -14,8 +10,6 @@ import MessageStore from '../core/messages/store'; export default { mixins: [LoaderMixin], components: { - tabs: Tabs, - tab: Tab, dropdown: Dropdown, }, data() { @@ -24,12 +18,11 @@ export default { error: false, success: false, message: undefined, - hasIfdo: false, + hasMetadata: false, + parsers: [], + selectedParser: null, }; }, - computed: { - // - }, methods: { handleSuccess() { this.error = false; @@ -37,7 +30,7 @@ export default { }, handleError(response) { this.success = false; - let knownError = response.body.errors && (response.body.errors.metadata || response.body.errors.ifdo_file || response.body.errors.file); + let knownError = response.body.errors && response.body.errors.file; if (knownError) { if (Array.isArray(knownError)) { this.error = knownError[0]; @@ -45,59 +38,41 @@ export default { this.error = knownError; } } else { - MessageStore.handleErrorResponse(response); + this.handleErrorResponse(response); } }, - submitCsv() { - this.$refs.csvInput.click(); - }, - uploadCsv(event) { - this.startLoading(); - let data = new FormData(); - data.append('metadata_csv', event.target.files[0]); - this.upload(data) - .then(this.handleSuccess, this.handleError) - .finally(this.finishLoading); - }, - submitIfdo() { - this.$refs.ifdoInput.click(); - }, - handleIfdo(event) { + handleFile(event) { this.startLoading(); let data = new FormData(); data.append('file', event.target.files[0]); - ParseIfdoFileApi.save(data) - .then(this.uploadIfdo) - .then(() => this.hasIfdo = true) + data.append('parser', this.selectedParser.parserClass); + MetadataApi.save({id: this.volumeId}, data) + .then(() => this.hasMetadata = true) .then(this.handleSuccess) .catch(this.handleError) .finally(this.finishLoading); }, - uploadIfdo(response) { - let ifdo = response.body; - let data = new FormData(); - data.append('ifdo_file', this.$refs.ifdoInput.files[0]); - data.append('metadata_text', ifdo.files.map(row => row.join(',')).join("\n")); - - return this.upload(data); - }, - upload(data) { - return MetadataApi.save({id: this.volumeId}, data); - }, - deleteIfdo() { + deleteFile() { this.startLoading(); - VolumeIfdoApi.delete({id: this.volumeId}) - .then(this.handleIfdoDeleted, MessageStore.handleErrorResponse) + MetadataApi.delete({id: this.volumeId}) + .then(this.handleFileDeleted, this.handleErrorResponse) .finally(this.finishLoading); }, - handleIfdoDeleted() { - this.hasIfdo = false; - MessageStore.success('The iFDO file was deleted.'); + handleFileDeleted() { + this.hasMetadata = false; + MessageStore.success('The metadata file was deleted.'); + }, + selectFile(parser) { + this.selectedParser = parser; + // Use $nextTick so the input element will have the appropriate MIME type + // filter from the selected parser. + this.$nextTick(() => this.$refs.fileInput.click()); }, }, created() { this.volumeId = biigle.$require('volumes.id'); - this.hasIfdo = biigle.$require('volumes.hasIfdo'); + this.hasMetadata = biigle.$require('volumes.hasMetadata'); + this.parsers = biigle.$require('volumes.parsers'); }, }; diff --git a/resources/assets/sass/label-trees/components/_labelTypeahead.scss b/resources/assets/sass/label-trees/components/_labelTypeahead.scss index 4436ac6f3..0bc0d5a1e 100644 --- a/resources/assets/sass/label-trees/components/_labelTypeahead.scss +++ b/resources/assets/sass/label-trees/components/_labelTypeahead.scss @@ -2,7 +2,7 @@ position: relative; // Negate the default padding of the list item of the typeahead. margin-left: -15px; - padding: $padding-small-vertical 22px; + padding: $padding-small-vertical 0 0 22px; .label-color { position: absolute; diff --git a/resources/assets/sass/volumes/components/_label-mapping.scss b/resources/assets/sass/volumes/components/_label-mapping.scss new file mode 100644 index 000000000..30a981d11 --- /dev/null +++ b/resources/assets/sass/volumes/components/_label-mapping.scss @@ -0,0 +1,36 @@ +.label-mapping { + margin-bottom: 1em; +} + +.label-mapping-item { + margin-bottom: $padding-small-vertical; + display: flex; + + &:hover { + color: white; + background-color: $gray-lighter; + border-radius: $border-radius-small; + } + + .label-tree-label__name { + padding-right: 0; + + &:hover { + cursor: inherit; + } + } + + .label-mapping-item-column { + flex: 1; + } + + .label-mapping-item-chevron { + flex: 0 0 40px; + padding: $padding-small-vertical 0; + text-align: center; + } + + .create-form .row:not(:last-child) { + padding-bottom: $padding-small-vertical; + } +} diff --git a/resources/assets/sass/volumes/components/_user-mapping.scss b/resources/assets/sass/volumes/components/_user-mapping.scss new file mode 100644 index 000000000..f186b788d --- /dev/null +++ b/resources/assets/sass/volumes/components/_user-mapping.scss @@ -0,0 +1,25 @@ +.user-mapping { + margin-bottom: 1em; +} + +.user-mapping-item { + margin-bottom: $padding-small-vertical; + padding: $padding-small-vertical $padding-small-horizontal; + padding-right: 0; + display: flex; + + &:hover { + color: white; + background-color: $gray-lighter; + border-radius: $border-radius-small; + } + + .user-mapping-item-column { + flex: 1; + } + + .user-mapping-item-chevron { + flex: 0 0 40px; + text-align: center; + } +} diff --git a/resources/assets/sass/volumes/main.scss b/resources/assets/sass/volumes/main.scss index 7d9821cc6..5a5ff2f2a 100644 --- a/resources/assets/sass/volumes/main.scss +++ b/resources/assets/sass/volumes/main.scss @@ -26,17 +26,6 @@ $volume-image-padding: 0.5em; background-color: $gray-lighter; } -.ifdo-icon { - width: 12px; - height: 12px; - display: inline-block; - margin-top: -3px; - - &.ifdo-icon--btn { - margin-top: -2px; - } -} - .volume-storage-disk-btn { overflow-x: hidden; text-overflow: ellipsis; @@ -56,6 +45,8 @@ $volume-image-padding: 0.5em; @import 'components/sorting-tab'; @import 'components/volume-image-grid-image'; @import 'components/file-label-list'; +@import 'components/label-mapping'; +@import 'components/user-mapping'; @import 'edit'; @import 'search'; diff --git a/resources/views/manual/index.blade.php b/resources/views/manual/index.blade.php index 3dc373773..e3ca7efbe 100644 --- a/resources/views/manual/index.blade.php +++ b/resources/views/manual/index.blade.php @@ -93,6 +93,14 @@ File labels are labels that are attached to whole images or videos.

+

+ Annotation and file label import +

+ +

+ Import annotations and file labels from metadata files. +

+

Files

diff --git a/resources/views/manual/tutorials/volumes/annotation-import.blade.php b/resources/views/manual/tutorials/volumes/annotation-import.blade.php new file mode 100644 index 000000000..8baf57104 --- /dev/null +++ b/resources/views/manual/tutorials/volumes/annotation-import.blade.php @@ -0,0 +1,44 @@ +@extends('manual.base') + +@section('manual-title') Annotation and file label import @stop + +@section('manual-content') +
+

+ Import annotations and file labels from metadata files. +

+

+ Some metadata file formats also support storing of annotation and/or file label information for images and videos. If you upload such a metadata file, the form to create a new volume will offer the options to import the annotations and/or file labels for the new volume, too. If you select one of these options, you are guided through a multi-step flow for the import. The individual steps are explained below. +

+ +

1. Create the volume

+

+ The import begins when you create the new volume and selected one of the options to import annotations and/or file labels. While the volume is created in the background, you are redirected to the next step. +

+ +

2. Choose annotation labels

+

+ This step only applies if you enabled the annotation import in step 1. In this step you are asked to choose all labels of annotations that should be imported. You can select only a subset of labels to import only certain annotations or you can select all labels to import all annotations. +

+ +

3. Choose file labels

+

+ This step only applies if you enabled the file label import in step 1. In this step you are asked to choose all file labels that should be imported. You can select only a subset of labels to import only certain file labels or you can select all labels. +

+ +

4. Label mapping

+

+ In this step you have to select one label from the BIIGLE database for each label of the metadata file that should be imported. Sometimes the metadata file stores unique identifiers for labels which enables BIIGLE to choose labels automatically. By default, you may choose any label from a label tree that is associated to the project to which the newly created volume (in step 1.) was added. You can also create new labels with pre-filled name (and maybe color) based on the metadata information. +

+ +

5. User mapping

+

+ In this step you have to select one user from the BIIGLE database for each user of the metadata file (i.e. creators of annotations and/or file labels) that should be imported. Sometimes the metadata file stores unique identifiers for users which enables BIIGLE to choose users automatically. You can choose any user of the BIIGLE instance (including your own). +

+ +

6. Finish import

+

+ This step verifies if all required information for the import is available or if you have to go back and provide additional information. When you click , you are redirected to the newly created volume (in step 1.) while the annotation and/or file label import proceeds in the background. +

+
+@endsection diff --git a/resources/views/manual/tutorials/volumes/file-metadata.blade.php b/resources/views/manual/tutorials/volumes/file-metadata.blade.php index 7c011d713..d64637bd2 100644 --- a/resources/views/manual/tutorials/volumes/file-metadata.blade.php +++ b/resources/views/manual/tutorials/volumes/file-metadata.blade.php @@ -11,7 +11,10 @@ BIIGLE supports metadata like the date and time of creation or the geo coordinates of a file. Every time a new image volume is created, BIIGLE attempts to automatically read the metadata from the EXIF information of JPEG files. This doesn't work for videos or if the images have another format than JPEG.

- In this case you can upload a metadata file. BIIGLE supports the iFDO standard for import of the metadata fields described below. Additionally, there is a custom CSV format for metadata import. The CSV file should use , as delimiter, " as enclosure and \ as escape characters. Please note the additional explanation of video metadata below. The following columns are supported (multiple synonyms exist for some colums, including the standard proposed in [1]): + In this case you can upload a metadata file. By default, BIIGLE supports a simple CSV file format for file metadata. More file formats supported by this instance may be found below. +

+

+ The CSV file should use , as delimiter, " as enclosure and \ as escape characters. Please note the additional explanation of video metadata below. The following columns are supported (multiple synonyms exist for some colums, including the standard proposed in [1]):

@@ -117,13 +120,13 @@ image_2.png,2016-12-19 17:09:31,52.215,28.501,-1502.5,28.25,2.1

- The metadata CSV file can be uploaded when a new volume is created. For existing volumes, metadata can be uploaded by volume admins on the volume edit page that you can reach with the button of the volume overview. + The metadata CSV file can be uploaded when a new volume is created. For existing volumes, metadata can be uploaded by volume admins on the volume edit page that you can reach with the button of the volume overview. This will replace any previously imported metadata.

Video metadata

- Video metadata can be imported in the "basic" or the "timestamped" form. The basic form is equivalent to image metadata where a video file can have at most one entry in the metadata CSV file. The timestamped form requires the taken_at column and allows to import many metadata values for different times of the same video. To import timestamped video metadata, add multiple rows with the same filename but different taken_at timestamp to the metadata CSV. Metadata will be ordered by timestamp and the earliest timestamp will be assumed to mark the beginning of the video. + Video metadata can be imported either in the "basic" or the "timestamped" form. The basic form is equivalent to image metadata where a video file can have at most one entry in the metadata CSV file. The timestamped form requires the taken_at column and allows to import many metadata values for different times of the same video. To import timestamped video metadata, add multiple rows with the same filename but different taken_at timestamp to the metadata CSV. Metadata will be ordered by timestamp and the earliest timestamp will be assumed to mark the beginning of the video.

Example: @@ -133,26 +136,18 @@ video_1.mp4,2016-12-19 17:09:00,52.112,28.001,-1500.5,30.25,2.6 video_1.mp4,2016-12-19 17:10:00,52.122,28.011,-1505.5,25.0,5.5 -

- Video metadata can be updated using the metadata upload of the volume edit page. Videos having "basic" metadata cannot be updated with "timestamped" metadata and vice versa. Timestamped metadata is merged with the information of the new CSV file if the taken_at timestamps of a video do not match exactly. For example, if a video already has the metadata of the example CSV shown above and now the following metadata file is uploaded: -

- 
-filename,taken_at,lng,lat
-video_1.mp4,2016-12-19 17:09:00,52.115,28.003
-video_1.mp4,2016-12-19 17:09:30,52.118,28.005
-
-

- The metadata of the video will be merged as follows: -

- 
-filename,taken_at,lng,lat,gps_altitude,distance_to_ground,area
-video_1.mp4,2016-12-19 17:09:00,52.115,28.003,-1500.5,30.25,2.6
-video_1.mp4,2016-12-19 17:09:30,52.118,28.005,null,null,null
-video_1.mp4,2016-12-19 17:10:00,52.122,28.011,-1505.5,25.0,5.5
-
-

- Note that the existing metadata with the same timestamp is updated with new metadata in the lng and lat fields. If metadata is missing for a timestamp, null is inserted. -

+
+
+

Additional metadata file formats

+ @if (empty(app('modules')->getViewMixins('metadataParsers'))) +

+ No additional file formats are supported. +

+ @else +
    + @mixin('metadataParsers') +
+ @endif

References

diff --git a/resources/views/volumes/create.blade.php b/resources/views/volumes/create.blade.php deleted file mode 100644 index 20cc92520..000000000 --- a/resources/views/volumes/create.blade.php +++ /dev/null @@ -1,271 +0,0 @@ -@extends('app') - -@section('title', 'Create new volume') - -@push('scripts') - -@endpush - -@section('content') -
-
-

New volume for {{ $project->name }}

-
- -
- - 1. Choose a media type - -
-
-
- -
-
- -
- -
- @if($errors->has('media_type')) - {{ $errors->first('media_type') }} - @endif -
-
- -
- - 2. Choose a name or import from file - -
-
- - @if($errors->has('name')) - {{ $errors->first('name') }} - @endif -
-
- - - - - - - -
-
-
- -
- -

Please import the metadata file again.

- - - - - @if ($errors->hasAny(['ifdo_file', 'metadata_csv', 'metadata'])) - @if ($errors->has('ifdo_file')) - {{ $errors->first('ifdo_file') }} - @endif - @if ($errors->has('metadata_csv')) - {{ $errors->first('metadata_csv') }} - @endif - @if ($errors->has('metadata')) - {{ $errors->first('metadata') }} - @endif - @else - Volume information and metadata can be imported from an iFDO YAML file or a metadata CSV file. Image metadata may be overridden by EXIF information. - @endif -
- -
- - 3. Choose a file source - - 0 files - - -
-
- @if ($offlineMode && $disks->isEmpty()) -
-
- Please configure available storage disks with the VOLUME_ADMIN_STORAGE_DISKS and VOLUME_EDITOR_STORAGE_DISKS environment variables. -
-
- @endif - @unless ($offlineMode) -
- -
- @endunless - @if ($userDisk) -
- -
- @endif - @if ($disks->count() > 1) -
- - - - -
- @elseif ($disks->count() === 1) -
- -
- @endif -
-
-
- - @unless ($offlineMode) -
-
- - - @if ($errors->has('url')) - {{ $errors->first('url') }} - @endif -
- -
-
- Remote locations for image volumes must support cross-origin resource sharing. -
-
- -
- -

- The filenames have been extracted from the provided metadata file. -

-
- -

- The filenames of the images in the volume directory formatted as comma separated values. Example: 1.jpg, 2.jpg, 3.jpg. The supported image file formats are: JPEG, PNG, WebP and TIFF. -

-
-
- -

- The filenames of the videos in the volume directory formatted as comma separated values. Example: 1.mp4, 2.mp4, 3.mp4. The supported video file formats are: MP4 (H.264) and WebM (VP8, VP9, AV1). -

-
- @if($errors->has('files')) - {{ $errors->first('files') }} - @endif -
-
- @endunless - -
- -

- loading available files... -

-

- - No files found. - @if (Route::has('create-storage-requests')) - Upload new files. - @endif - - - Select a directory or files below. All selected files will be used for the new volume. - - Only files with a supported image file format are shown (JPEG, PNG, WebP and TIFF). - Only files with a supported video file format are shown (MP4 (H.264) and WebM (VP8, VP9, AV1)). -

- - - - - - - - @if ($errors->has('url')) -
- {{ $errors->first('url') }} -
- @endif - @if ($errors->has('files')) -
- {{ $errors->first('files') }} -
- @endif -
- -
-
- Most browsers do not support the TIFF format. Only use it for very large images with more than {{config('image.tiles.threshold')}} pixels at one edge, as these will be automatically converted by BIIGLE. -
-
- -
- - 4. Set a handle or DOI (optional) - - -
- - - A handle or DOI to be associated with the volume. - - @if($errors->has('handle')) - {{ $errors->first('handle') }} - @endif -
-
- -
- - - Cancel - -
- -
-
-@endsection diff --git a/resources/views/volumes/create/annotationLabels.blade.php b/resources/views/volumes/create/annotationLabels.blade.php new file mode 100644 index 000000000..a58824d09 --- /dev/null +++ b/resources/views/volumes/create/annotationLabels.blade.php @@ -0,0 +1,60 @@ +@extends('app') + +@section('title', 'Select metadata anntation labels to import') + +@push('scripts') + +@endpush + +@section('content') + +
+
+

Choose labels to filter metadata annotations

+

+ If you wish to import only annotations with certain labels from the metadata file, you can choose these labels here. By default, all labels are selected. With Alt+click you can select and deselect all labels. +

+
id}/annotation-labels") }}" v-on:submit="startLoading"> + +
+ +
+ +
+
+ +
+ @if ($errors->has('labels')) + {{ $errors->first('labels') }} + @endif +
+ + +
+

+ All labels are selected. +

+

+ labels are selected. +

+
+ +
+ + + + + import_file_labels) href="{{ route('pending-volume-file-labels', $pv->id) }}" @else href="{{ route('pending-volume-label-map', $pv->id) }}" @endif class="btn btn-success pull-right">Continue + +
+ +
id}") }}" v-on:submit="startLoading"> + + + + +
+
+@endsection diff --git a/resources/views/volumes/create/fileLabels.blade.php b/resources/views/volumes/create/fileLabels.blade.php new file mode 100644 index 000000000..718db008d --- /dev/null +++ b/resources/views/volumes/create/fileLabels.blade.php @@ -0,0 +1,60 @@ +@extends('app') + +@section('title', 'Select metadata file labels to import') + +@push('scripts') + +@endpush + +@section('content') + +
+
+

Choose file labels to filter metadata

+

+ If you wish to import only certain file labels from the metadata file, you can choose these labels here. By default, all labels are selected. With Alt+click you can select and deselect all labels. +

+
id}/file-labels") }}" v-on:submit="startLoading"> + +
+ +
+ +
+
+ +
+ @if ($errors->has('labels')) + {{ $errors->first('labels') }} + @endif +
+ + +
+

+ All labels are selected. +

+

+ labels are selected. +

+
+ +
+ + + + + Continue + +
+ +
id}") }}" v-on:submit="startLoading"> + + + + +
+
+@endsection diff --git a/resources/views/volumes/create/finish.blade.php b/resources/views/volumes/create/finish.blade.php new file mode 100644 index 000000000..d488a0062 --- /dev/null +++ b/resources/views/volumes/create/finish.blade.php @@ -0,0 +1,90 @@ +@extends('app') + +@section('title', 'Finish the metadata import') + +@section('content') + +
+
+

Finish metadata import

+ +
    + @if ($pv->import_annotations) + @if (empty($pv->only_annotation_labels)) +
  • + All annotations will be imported + +
    • + @else +
  • + Annotations with selected labels will be imported + +
    • + @endif + @endif + @if ($pv->import_file_labels) + @if (empty($pv->only_file_labels)) +
  • + All file labels will be imported + +
    • + @else +
  • + Selected file labels will be imported + +
    • + @endif + @endif + @if ($labelMapOk) +
  • + All labels are mapped + +
    • + @else +
  • + Labels are not mapped + +
    • + @endif + @if ($userMapOk) +
  • + All users are mapped + +
    • + @else +
  • + Users are not mapped + +
    • + @endif +
+ + @if (!$labelMapOk) + Back to label mapping + + + @elseif (!$userMapOk) + Back to user mapping + + + @else +
id}/import") }}" v-on:submit="startLoading"> +
+ @if ($errors->has('id')) + {{ $errors->first('id') }} + @endif + + + + +
+ + @endif +
id}") }}" v-on:submit="startLoading"> + + + + +
+
+@endsection diff --git a/resources/views/volumes/create/labelMap.blade.php b/resources/views/volumes/create/labelMap.blade.php new file mode 100644 index 000000000..0e0f7af6c --- /dev/null +++ b/resources/views/volumes/create/labelMap.blade.php @@ -0,0 +1,71 @@ +@extends('app') + +@section('title', 'Select the metadata import label mapping') + +@push('scripts') + +@endpush + +@section('content') + +
+
+

Metadata import label mapping

+

+ Each label from the metadata file must be mapped to a label in the BIIGLE database. Some labels can be mapped automatically. The rest must be mapped manually. You can also create new labels based on the metadata information. +

+
id}/label-map") }}" v-on:submit="startLoading"> + +
+ + @if ($errors->has('label_map')) + {{ $errors->first('label_map') }} + @endif +
+ + + +
+

+ All labels are mapped. +

+

+ labels must be mapped. +

+
+ +
+ + + + + + + + +
+ +
id}") }}" v-on:submit="startLoading"> + + + + +
+
+@endsection diff --git a/resources/views/volumes/create/step1.blade.php b/resources/views/volumes/create/step1.blade.php new file mode 100644 index 000000000..4064640d7 --- /dev/null +++ b/resources/views/volumes/create/step1.blade.php @@ -0,0 +1,79 @@ +@extends('app') + +@section('title', 'Start creating a new volume') + +@push('scripts') + +@endpush + +@section('content') + +
+
+

New volume for {{ $project->name }}

+
id}/pending-volumes") }}" enctype="multipart/form-data" v-on:submit="startLoading"> +
+ + Choose a media type + +
+
+
+ + +
+
+ +
+
+ @if ($errors->has('media_type')) +

{{ $errors->first('media_type') }}

+ @endif +

+ The media type determines the type of files that can be included in the volume. Each type requires different annotation tools. +

+ +
+
+
+ + Select a metadata file (optional) + +
+

+ + + + +

+ + + + @if ($errors->has('metadata_file')) +

{{ $errors->first('metadata_file') }}

+ @endif + + @if ($errors->has('metadata_parser')) +

{{ $errors->first('metadata_parser') }}

+ @endif +

+ By default, BIIGLE supports a CSV metadata format. Other supported formats may be listed in the manual. Image metadata may be overridden by EXIF information during the creation of the volume. +

+
+
+
+ + Cancel + +
+ +
+
+@endsection diff --git a/resources/views/volumes/create/step2.blade.php b/resources/views/volumes/create/step2.blade.php new file mode 100644 index 000000000..4c0a58bd7 --- /dev/null +++ b/resources/views/volumes/create/step2.blade.php @@ -0,0 +1,258 @@ +@extends('app') + +@section('title', 'Finish creating a new volume') + +@push('scripts') + +@endpush + +@section('content') + +
+
+

New {{$mediaType}} volume for {{ $project->name }}

+
id}") }}" v-on:submit="startLoading"> +
+ + Choose a volume name + +
+ + @if ($errors->has('name')) + {{ $errors->first('name') }} + @endif +
+
+ +
+ + Choose a file source + + 0 files + + +
+
+ @if ($offlineMode && $disks->isEmpty()) +
+
+ Please configure available storage disks with the VOLUME_ADMIN_STORAGE_DISKS and VOLUME_EDITOR_STORAGE_DISKS environment variables. +
+
+ @endif + @unless ($offlineMode) +
+ + +
+ @endunless + @if ($userDisk) +
+ +
+ @endif + @if ($disks->count() > 1) +
+ + + + +
+ @elseif ($disks->count() === 1) +
+ +
+ @endif +
+
+
+ + @unless ($offlineMode) +
+
+ + + @if ($errors->has('url')) + {{ $errors->first('url') }} + @endif +
+ + @if ($isImageMediaType) +
+
+ Remote locations for image volumes must support cross-origin resource sharing. +
+
+ @endif + +
+ + @if ($isImageMediaType) + +

+ The filenames of the images in the volume directory formatted as comma separated values. Example: 1.jpg, 2.jpg, 3.jpg. The supported image file formats are: JPEG, PNG, WebP and TIFF. +

+ @else + +

+ The filenames of the videos in the volume directory formatted as comma separated values. Example: 1.mp4, 2.mp4, 3.mp4. The supported video file formats are: MP4 (H.264) and WebM (VP8, VP9, AV1). +

+ @endif + @if($errors->has('files')) + {{ $errors->first('files') }} + @endif +
+
+ @endunless + +
+ +

+ loading available files... +

+

+ + No files found. + @if (Route::has('create-storage-requests')) + Upload new files. + @endif + + + Select a directory or files below. All selected files will be used for the new volume. + + @if ($isImageMediaType) + Only files with a supported image file format are shown (JPEG, PNG, WebP and TIFF). + @else + Only files with a supported video file format are shown (MP4 (H.264) and WebM (VP8, VP9, AV1)). + @endif +

+ + + + + + + + @if ($errors->has('url')) +
+ {{ $errors->first('url') }} +
+ @endif + @if ($errors->has('files')) +
+ {{ $errors->first('files') }} +
+ @endif +
+ +
+
+ Most browsers do not support the TIFF format. Only use it for very large images with more than {{config('image.tiles.threshold')}} pixels at one edge, as these will be automatically converted by BIIGLE. +
+
+ +
+
+ The chosen files could not be found in the uploaded metadata file! +
+
+ +
+ + Set a handle or DOI (optional) + + +
+ + + A handle or DOI to be associated with the volume. + + @if ($errors->has('handle')) + {{ $errors->first('handle') }} + @endif +
+
+ + @if ($hasAnnotations || $hasFileLabels) +
+ + Import more metadata (optional) + + +
+ @if ($hasAnnotations) +
+ + + @if ($errors->has('import_annotations')) + {{ $errors->first('import_annotations') }} + @endif +
+ @endif + + @if ($hasFileLabels) +
+ + + @if ($errors->has('import_file_labels')) + {{ $errors->first('import_file_labels') }} + @endif +
+ @endif +
+ + + If you import more metadata, there will be additional steps after the volume was created. + +
+ @endif + +
+ + + + + + +
+ +
id}") }}" v-on:submit="startLoading"> + + + +
+
+@endsection diff --git a/resources/views/volumes/create/userMap.blade.php b/resources/views/volumes/create/userMap.blade.php new file mode 100644 index 000000000..9a3ab26bf --- /dev/null +++ b/resources/views/volumes/create/userMap.blade.php @@ -0,0 +1,70 @@ +@extends('app') + +@section('title', 'Select the metadata import user mapping') + +@push('scripts') + +@endpush + +@section('content') + +
+
+

Metadata import user mapping

+

+ Each user from the metadata file must be mapped to a user in the BIIGLE database. Some labels can be mapped automatically. The rest must be mapped manually. +

+
id}/user-map") }}" v-on:submit="startLoading"> + +
+ + @if ($errors->has('user_map')) + {{ $errors->first('user_map') }} + @endif +
+ + + +
+

+ All users are mapped. +

+

+ users must be mapped. +

+
+ +
+ + + + + + + + +
+ +
id}") }}" v-on:submit="startLoading"> + + + + +
+
+@endsection diff --git a/resources/views/volumes/edit.blade.php b/resources/views/volumes/edit.blade.php index 9507de6db..690be059c 100644 --- a/resources/views/volumes/edit.blade.php +++ b/resources/views/volumes/edit.blade.php @@ -6,7 +6,8 @@ biigle.$declare('volumes.id', {!! $volume->id !!}); biigle.$declare('volumes.annotationSessions', {!! $annotationSessions !!}); biigle.$declare('volumes.type', '{!! $type !!}'); - biigle.$declare('volumes.hasIfdo', '{!! $volume->hasIfdo() !!}'); + biigle.$declare('volumes.hasMetadata', '{!! $volume->hasMetadata() !!}'); + biigle.$declare('volumes.parsers', {!! $parsers !!}); @mixin('volumesEditScripts') @endpush diff --git a/resources/views/volumes/edit/metadata.blade.php b/resources/views/volumes/edit/metadata.blade.php index 56f1769d4..37d83d876 100644 --- a/resources/views/volumes/edit/metadata.blade.php +++ b/resources/views/volumes/edit/metadata.blade.php @@ -7,47 +7,39 @@ @endif - - + +
- - -

- Upload an iFDO file to attach it to the volume and update the @if ($volume->isImageVolume()) image @else video @endif metadata. -

-
-
- - -
- - - -

- Upload a CSV file to update the metadata of the @if ($volume->isImageVolume()) images @else videos @endif of this volume. -

-
-
- - -
- - - +

+ Upload a metadata file to attach it to the volume and update the @if ($volume->isImageVolume()) image @else video @endif metadata. +

+

+ + + + +

+ + +
- The @if ($volume->isImageVolume()) image @else video @endif metadata was successfully updated. + The @if ($volume->isImageVolume()) image @else video @endif metadata file was successfully updated.

Learn more about @if ($volume->isImageVolume()) image @else video @endif metadata and the file formats in the manual. diff --git a/resources/views/volumes/partials/handleIndicator.blade.php b/resources/views/volumes/partials/handleIndicator.blade.php index 1ccb77958..801371f0b 100644 --- a/resources/views/volumes/partials/handleIndicator.blade.php +++ b/resources/views/volumes/partials/handleIndicator.blade.php @@ -1,3 +1,3 @@ @if ($volume->handle) - + @endif diff --git a/resources/views/volumes/partials/ifdoIndicator.blade.php b/resources/views/volumes/partials/ifdoIndicator.blade.php deleted file mode 100644 index f51d344d1..000000000 --- a/resources/views/volumes/partials/ifdoIndicator.blade.php +++ /dev/null @@ -1,5 +0,0 @@ -@if ($volume->hasIfdo(true)) - id}/ifdo")}}" class="btn btn-default btn-xs" title="Download the iFDO attached to this volume"> - iFDO - -@endif diff --git a/resources/views/volumes/partials/metadataIndicator.blade.php b/resources/views/volumes/partials/metadataIndicator.blade.php new file mode 100644 index 000000000..69f9d613f --- /dev/null +++ b/resources/views/volumes/partials/metadataIndicator.blade.php @@ -0,0 +1,5 @@ +@if ($volume->hasMetadata()) + id}/metadata")}}" class="btn btn-default btn-xs" title="Download the metadata file attached to this volume"> + + +@endif diff --git a/resources/views/volumes/show.blade.php b/resources/views/volumes/show.blade.php index 467b0612d..e28c73cea 100644 --- a/resources/views/volumes/show.blade.php +++ b/resources/views/volumes/show.blade.php @@ -36,7 +36,7 @@ @section('navbar')

- @include('volumes.partials.projectsBreadcrumb') / {{$volume->name}} ({{ $fileIds->count() }} {{$type}}s) @include('volumes.partials.annotationSessionIndicator') @include('volumes.partials.handleIndicator') @include('volumes.partials.ifdoIndicator') + @include('volumes.partials.projectsBreadcrumb') / {{$volume->name}} ({{ $fileIds->count() }} {{$type}}s) @include('volumes.partials.annotationSessionIndicator') @include('volumes.partials.handleIndicator') @include('volumes.partials.metadataIndicator')
@endsection diff --git a/routes/api.php b/routes/api.php index 1c4997bc0..668a41d87 100644 --- a/routes/api.php +++ b/routes/api.php @@ -149,6 +149,17 @@ 'only' => ['update', 'destroy'], ]); +$router->resource('pending-volumes', 'PendingVolumeController', [ + 'only' => ['update', 'destroy'], + 'parameters' => ['pending-volumes' => 'id'], +]); + +$router->put('pending-volumes/{id}/annotation-labels', 'PendingVolumeImportController@updateAnnotationLabels'); +$router->put('pending-volumes/{id}/file-labels', 'PendingVolumeImportController@updateFileLabels'); +$router->put('pending-volumes/{id}/label-map', 'PendingVolumeImportController@updateLabelMap'); +$router->put('pending-volumes/{id}/user-map', 'PendingVolumeImportController@updateUserMap'); +$router->post('pending-volumes/{id}/import', 'PendingVolumeImportController@storeImport'); + $router->resource('projects', 'ProjectController', [ 'only' => ['index', 'show', 'update', 'store', 'destroy'], 'parameters' => ['projects' => 'id'], @@ -170,6 +181,11 @@ 'parameters' => ['projects' => 'id', 'label-trees' => 'id2'], ]); +$router->resource('projects.pending-volumes', 'PendingVolumeController', [ + 'only' => ['store'], + 'parameters' => ['projects' => 'id'], +]); + $router->get( 'projects/pinned', 'UserPinnedProjectController@index' @@ -307,10 +323,6 @@ $router->get('videos/{disk}', 'BrowserController@indexVideos'); }); - $router->post('parse-ifdo', [ - 'uses' => 'ParseIfdoController@store', - ]); - $router->get('{id}/files/filter/labels', [ 'uses' => 'Filters\AnyFileLabelController@index', ]); @@ -352,12 +364,12 @@ 'uses' => 'MetadataController@store', ]); - $router->get('{id}/ifdo', [ - 'uses' => 'IfdoController@show', + $router->get('{id}/metadata', [ + 'uses' => 'MetadataController@show', ]); - $router->delete('{id}/ifdo', [ - 'uses' => 'IfdoController@destroy', + $router->delete('{id}/metadata', [ + 'uses' => 'MetadataController@destroy', ]); $router->get('{id}/users', [ diff --git a/routes/web.php b/routes/web.php index 0970d1c79..bdf0911a7 100644 --- a/routes/web.php +++ b/routes/web.php @@ -247,6 +247,38 @@ ]); }); + $router->group(['namespace' => 'Volumes', 'prefix' => 'pending-volumes'], function ($router) { + $router->get('{id}', [ + 'as' => 'pending-volume', + 'uses' => 'PendingVolumeController@show', + ]); + + $router->get('{id}/annotation-labels', [ + 'as' => 'pending-volume-annotation-labels', + 'uses' => 'PendingVolumeController@showAnnotationLabels', + ]); + + $router->get('{id}/file-labels', [ + 'as' => 'pending-volume-file-labels', + 'uses' => 'PendingVolumeController@showFileLabels', + ]); + + $router->get('{id}/label-map', [ + 'as' => 'pending-volume-label-map', + 'uses' => 'PendingVolumeController@showLabelMap', + ]); + + $router->get('{id}/user-map', [ + 'as' => 'pending-volume-user-map', + 'uses' => 'PendingVolumeController@showUserMap', + ]); + + $router->get('{id}/finish', [ + 'as' => 'pending-volume-finish', + 'uses' => 'PendingVolumeController@showFinish', + ]); + }); + $router->group(['namespace' => 'Volumes', 'prefix' => 'volumes'], function ($router) { $router->get('create', [ 'as' => 'create-volume', diff --git a/tests/files/image-ifdo.yaml b/tests/files/image-ifdo.yaml deleted file mode 100644 index 5d14b386b..000000000 --- a/tests/files/image-ifdo.yaml +++ /dev/null @@ -1,37 +0,0 @@ -image-set-header: - image-acquisition: photo - image-area-square-meter: 5.0 - image-capture-mode: mixed - image-coordinate-reference-system: EPSG:4326 - image-deployment: survey - image-event: SO268-2_100-1_OFOS - image-illumination: artificial - image-latitude: 11.8581802 - image-license: CC-BY - image-longitude: -117.0214864 - image-marine-zone: seafloor - image-meters-above-ground: 2 - image-navigation: beacon - image-project: SO268 - image-quality: raw - image-resolution: mm - image-scale-reference: laser marker - image-set-data-handle: 20.500.12085/d7546c4b-307f-4d42-8554-33236c577450@data - image-set-handle: 20.500.12085/d7546c4b-307f-4d42-8554-33236c577450 - image-set-metadata-handle: 20.500.12085/d7546c4b-307f-4d42-8554-33236c577450@metadata - image-set-name: SO268 SO268-2_100-1_OFOS SO_CAM-1_Photo_OFOS - image-set-uuid: d7546c4b-307f-4d42-8554-33236c577450 - image-spectral-resolution: rgb -image-set-items: - SO268-2_100-1_OFOS_SO_CAM-1_20190406_042927.JPG: - image-datetime: '2019-04-06 04:29:27.000000' - image-depth: 2248.0 - image-camera-yaw-degrees: 20 - SO268-2_100-1_OFOS_SO_CAM-1_20190406_052726.JPG: - - image-datetime: '2019-04-06 05:27:26.000000' - image-altitude: -4129.6 - image-latitude: 11.8582192 - image-longitude: -117.0214286 - image-area-square-meter: 5.1 - image-meters-above-ground: 2.1 - image-camera-yaw-degrees: 21 diff --git a/tests/files/image-metadata-invalid.csv b/tests/files/image-metadata-invalid.csv new file mode 100644 index 000000000..0c0937c59 --- /dev/null +++ b/tests/files/image-metadata-invalid.csv @@ -0,0 +1,2 @@ +filename,taken_at,lng,lat,gps_altitude,distance_to_ground,area,yaw +abc.jpg,2016-12-19 12:27:00,52.220,28.123,-1500,10,2.6,999 diff --git a/tests/files/image-metadata-strange-encoding.csv b/tests/files/image-metadata-strange-encoding.csv new file mode 100644 index 000000000..bd460d2c8 --- /dev/null +++ b/tests/files/image-metadata-strange-encoding.csv @@ -0,0 +1,2 @@ + taken_at,filename,gps_altitude +2023-03-26 12:40:47,my-image ,-10 diff --git a/tests/files/video-ifdo.yaml b/tests/files/video-ifdo.yaml deleted file mode 100644 index 6cc8bfd09..000000000 --- a/tests/files/video-ifdo.yaml +++ /dev/null @@ -1,19 +0,0 @@ -image-set-header: - image-acquisition: video - image-area-square-meter: 5.0 - image-latitude: 11.8581802 - image-longitude: -117.0214864 - image-meters-above-ground: 2 - image-set-data-handle: 20.500.12085/d7546c4b-307f-4d42-8554-33236c577450@data - image-set-handle: 20.500.12085/d7546c4b-307f-4d42-8554-33236c577450 - image-set-metadata-handle: 20.500.12085/d7546c4b-307f-4d42-8554-33236c577450@metadata - image-set-name: SO268 SO268-2_100-1_OFOS SO_CAM-1_Photo_OFOS - image-set-uuid: d7546c4b-307f-4d42-8554-33236c577450 -image-set-items: - video1.mp4: - - image-datetime: '2019-04-06 04:29:27.000000' - image-depth: 2248.0 - image-camera-yaw-degrees: 20 - - image-datetime: '2019-04-06 04:30:27.000000' - image-depth: 2250.0 - image-camera-yaw-degrees: 21 diff --git a/tests/files/video-metadata-incorrect-encoding.csv b/tests/files/video-metadata-incorrect-encoding.csv deleted file mode 100644 index c74083723..000000000 --- a/tests/files/video-metadata-incorrect-encoding.csv +++ /dev/null @@ -1,2 +0,0 @@ -filename,taken_at,gps_altitude - my-video ,2023-03-26 12:40:47,-10 diff --git a/tests/files/video-metadata-invalid.csv b/tests/files/video-metadata-invalid.csv new file mode 100644 index 000000000..0320a04f1 --- /dev/null +++ b/tests/files/video-metadata-invalid.csv @@ -0,0 +1,3 @@ +filename,taken_at,lng,lat,gps_altitude,distance_to_ground,area,yaw +abc.mp4,2016-12-19 12:27:00,52.220,28.123,-1500,10,2.6,999 +abc.mp4,2016-12-19 12:28:00,52.230,28.133,-1505,5,1.6,181 diff --git a/tests/files/video-metadata-strange-encoding.csv b/tests/files/video-metadata-strange-encoding.csv new file mode 100644 index 000000000..9ab5ea006 --- /dev/null +++ b/tests/files/video-metadata-strange-encoding.csv @@ -0,0 +1,2 @@ + taken_at,filename,gps_altitude +2023-03-26 12:40:47,my-video ,-10 diff --git a/tests/php/Http/Controllers/Api/PendingVolumeControllerTest.php b/tests/php/Http/Controllers/Api/PendingVolumeControllerTest.php new file mode 100644 index 000000000..9c36a4919 --- /dev/null +++ b/tests/php/Http/Controllers/Api/PendingVolumeControllerTest.php @@ -0,0 +1,876 @@ +project()->id; + $this->doTestApiRoute('POST', "/api/v1/projects/{$id}/pending-volumes"); + + $this->beEditor(); + $this->post("/api/v1/projects/{$id}/pending-volumes")->assertStatus(403); + + $this->beAdmin(); + // Missing arguments. + $this->json('POST', "/api/v1/projects/{$id}/pending-volumes")->assertStatus(422); + + // Incorrect media type. + $this->json('POST', "/api/v1/projects/{$id}/pending-volumes", [ + 'media_type' => 'whatever', + ])->assertStatus(422); + + $this->json('POST', "/api/v1/projects/{$id}/pending-volumes", [ + 'media_type' => 'image', + ])->assertStatus(201); + + $pv = PendingVolume::where('project_id', $id)->first(); + $this->assertEquals(MediaType::imageId(), $pv->media_type_id); + $this->assertEquals($this->admin()->id, $pv->user_id); + } + + public function testStoreTwice() + { + $this->beAdmin(); + $id = $this->project()->id; + $this->json('POST', "/api/v1/projects/{$id}/pending-volumes", [ + 'media_type' => 'image', + ])->assertStatus(201); + + $this->json('POST', "/api/v1/projects/{$id}/pending-volumes", [ + 'media_type' => 'image', + ])->assertStatus(422); + } + + public function testStoreVideo() + { + $this->beAdmin(); + $id = $this->project()->id; + $this->json('POST', "/api/v1/projects/{$id}/pending-volumes", [ + 'media_type' => 'video', + ])->assertStatus(201); + } + + public function testStoreImageWithFile() + { + $disk = Storage::fake('pending-metadata'); + $csv = __DIR__."/../../../../files/image-metadata.csv"; + $file = new UploadedFile($csv, 'metadata.csv', 'text/csv', null, true); + + $id = $this->project()->id; + $this->beAdmin(); + $this->json('POST', "/api/v1/projects/{$id}/pending-volumes", [ + 'media_type' => 'image', + 'metadata_parser' => ImageCsvParser::class, + 'metadata_file' => $file, + ])->assertStatus(201); + + $pv = PendingVolume::where('project_id', $id)->first(); + $this->assertEquals(ImageCsvParser::class, $pv->metadata_parser); + $this->assertNotNull($pv->metadata_file_path); + $disk->assertExists($pv->metadata_file_path); + } + + public function testStoreImageWithFileUnknown() + { + $disk = Storage::fake('pending-metadata'); + $csv = __DIR__."/../../../../files/test.mp4"; + $file = new UploadedFile($csv, 'metadata.csv', 'text/csv', null, true); + + $id = $this->project()->id; + $this->beAdmin(); + $this->json('POST', "/api/v1/projects/{$id}/pending-volumes", [ + 'media_type' => 'image', + 'metadata_parser' => ImageCsvParser::class, + 'metadata_file' => $file, + ])->assertStatus(422); + } + + public function testStoreImageWithFileInvalid() + { + $disk = Storage::fake('pending-metadata'); + $csv = __DIR__."/../../../../files/image-metadata-invalid.csv"; + $file = new UploadedFile($csv, 'metadata.csv', 'text/csv', null, true); + + $id = $this->project()->id; + $this->beAdmin(); + $this->json('POST', "/api/v1/projects/{$id}/pending-volumes", [ + 'media_type' => 'image', + 'metadata_parser' => ImageCsvParser::class, + 'metadata_file' => $file, + ])->assertStatus(422); + } + + public function testStoreImageWithParserUnknown() + { + $disk = Storage::fake('pending-metadata'); + $csv = __DIR__."/../../../../files/image-metadata.csv"; + $file = new UploadedFile($csv, 'metadata.csv', 'text/csv', null, true); + + $id = $this->project()->id; + $this->beAdmin(); + $this->json('POST', "/api/v1/projects/{$id}/pending-volumes", [ + 'media_type' => 'image', + 'metadata_parser' => 'my-parser', + 'metadata_file' => $file, + ])->assertStatus(422); + } + + public function testStoreVideoWithFile() + { + $disk = Storage::fake('pending-metadata'); + $csv = __DIR__."/../../../../files/video-metadata.csv"; + $file = new UploadedFile($csv, 'metadata.csv', 'text/csv', null, true); + + $id = $this->project()->id; + $this->beAdmin(); + $this->json('POST', "/api/v1/projects/{$id}/pending-volumes", [ + 'media_type' => 'video', + 'metadata_parser' => VideoCsvParser::class, + 'metadata_file' => $file, + ])->assertStatus(201); + + $pv = PendingVolume::where('project_id', $id)->first(); + $this->assertEquals(VideoCsvParser::class, $pv->metadata_parser); + $this->assertNotNull($pv->metadata_file_path); + $disk->assertExists($pv->metadata_file_path); + } + + public function testStoreVideoWithFileUnknown() + { + $disk = Storage::fake('pending-metadata'); + $csv = __DIR__."/../../../../files/test.mp4"; + $file = new UploadedFile($csv, 'metadata.csv', 'text/csv', null, true); + + $id = $this->project()->id; + $this->beAdmin(); + $this->json('POST', "/api/v1/projects/{$id}/pending-volumes", [ + 'media_type' => 'video', + 'metadata_parser' => VideoCsvParser::class, + 'metadata_file' => $file, + ])->assertStatus(422); + } + + public function testStoreVideoWithFileInvalid() + { + $disk = Storage::fake('pending-metadata'); + $csv = __DIR__."/../../../../files/video-metadata-invalid.csv"; + $file = new UploadedFile($csv, 'metadata.csv', 'text/csv', null, true); + + $id = $this->project()->id; + $this->beAdmin(); + $this->json('POST', "/api/v1/projects/{$id}/pending-volumes", [ + 'media_type' => 'video', + 'metadata_parser' => VideoCsvParser::class, + 'metadata_file' => $file, + ])->assertStatus(422); + } + + public function testStoreVideoWithParserUnknown() + { + $disk = Storage::fake('pending-metadata'); + $csv = __DIR__."/../../../../files/video-metadata.csv"; + $file = new UploadedFile($csv, 'metadata.csv', 'text/csv', null, true); + + $id = $this->project()->id; + $this->beAdmin(); + $this->json('POST', "/api/v1/projects/{$id}/pending-volumes", [ + 'media_type' => 'video', + 'metadata_parser' => 'my-parser', + 'metadata_file' => $file, + ])->assertStatus(422); + } + + public function testStoreFromUI() + { + $id = $this->project()->id; + + $this->beAdmin(); + $response = $this->post("/api/v1/projects/{$id}/pending-volumes", [ + 'media_type' => 'image', + ]); + + $pv = PendingVolume::first(); + + $response->assertRedirectToRoute('pending-volume', $pv->id); + } + + public function testUpdateImages() + { + config(['volumes.editor_storage_disks' => ['test']]); + $disk = Storage::fake('test'); + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->editor()->id, + ]); + $id = $pv->id; + $this->doTestApiRoute('PUT', "/api/v1/pending-volumes/{$id}"); + + $this->beEditor(); + $this->putJson("/api/v1/pending-volumes/{$id}")->assertStatus(403); + + $this->beAdmin(); + // Does not own the pending volume. + $this->putJson("/api/v1/pending-volumes/{$id}")->assertStatus(403); + + $pv->update(['user_id' => $this->admin()->id]); + // mssing arguments + $this->putJson("/api/v1/pending-volumes/{$id}")->assertStatus(422); + + // invalid url format + $this->putJson("/api/v1/pending-volumes/{$id}", [ + 'name' => 'my volume no. 1', + 'url' => 'test', + 'files' => ['1.jpg', '2.jpg'], + ])->assertStatus(422); + + // unknown storage disk + $this->putJson("/api/v1/pending-volumes/{$id}", [ + 'name' => 'my volume no. 1', + 'url' => 'random', + 'files' => ['1.jpg', '2.jpg'], + ])->assertStatus(422); + + // images directory dows not exist in storage disk + $this->putJson("/api/v1/pending-volumes/{$id}", [ + 'name' => 'my volume no. 1', + 'url' => 'test://images', + 'files' => ['1.jpg', '2.jpg'], + ])->assertStatus(422); + + $disk->makeDirectory('images'); + $disk->put('images/1.jpg', 'abc'); + $disk->put('images/2.jpg', 'abc'); + $disk->put('images/1.bmp', 'abc'); + + // images array is empty + $this->putJson("/api/v1/pending-volumes/{$id}", [ + 'name' => 'my volume no. 1', + 'url' => 'test://images', + 'files' => [], + ])->assertStatus(422); + + // error because of duplicate image + $this->putJson("/api/v1/pending-volumes/{$id}", [ + 'name' => 'my volume no. 1', + 'url' => 'test://images', + 'files' => ['1.jpg', '1.jpg'], + ])->assertStatus(422); + + // error because of unsupported image format + $this->putJson("/api/v1/pending-volumes/{$id}", [ + 'name' => 'my volume no. 1', + 'url' => 'test://images', + 'files' => ['1.bmp'], + ])->assertStatus(422); + + // Image filename too long. + $this->putJson("/api/v1/pending-volumes/{$id}", [ + 'name' => 'my volume no. 1', + 'url' => 'test://images', + 'files' => ['aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.jpg'], + ])->assertStatus(422); + + $response = $this->putJson("/api/v1/pending-volumes/{$id}", [ + 'name' => 'my volume no. 1', + 'url' => 'test://images', + // Elements should be sanitized and empty elements should be discarded + 'files' => ['" 1.jpg"', '', '\'2.jpg\' ', '', ''], + ])->assertSuccessful(); + $content = $response->getContent(); + $this->assertStringStartsWith('{', $content); + $this->assertStringEndsWith('}', $content); + + $id = json_decode($content)->volume_id; + Queue::assertPushed(CreateNewImagesOrVideos::class, function ($job) use ($id) { + $this->assertEquals($id, $job->volume->id); + $this->assertContains('1.jpg', $job->filenames); + $this->assertContains('2.jpg', $job->filenames); + $this->assertCount(2, $job->filenames); + + return true; + }); + + $this->assertNull($pv->fresh()); + + $this->assertEquals(1, $this->project()->volumes()->count()); + $volume = $this->project()->volumes()->first(); + $this->assertEquals('my volume no. 1', $volume->name); + $this->assertEquals('test://images', $volume->url); + $this->assertEquals(MediaType::imageId(), $volume->media_type_id); + } + + public function testUpdateImagesWithMetadata() + { + $pendingMetaDisk = Storage::fake('pending-metadata'); + $metaDisk = Storage::fake('metadata'); + $fileDisk = Storage::fake('test'); + config(['volumes.editor_storage_disks' => ['test']]); + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + 'metadata_file_path' => 'mymeta.csv', + 'metadata_parser' => ImageCsvParser::class, + ]); + $id = $pv->id; + $pendingMetaDisk->put('mymeta.csv', 'abc'); + + $fileDisk->makeDirectory('images'); + $fileDisk->put('images/1.jpg', 'abc'); + + $this->beAdmin(); + $this->putJson("/api/v1/pending-volumes/{$id}", [ + 'name' => 'my volume no. 1', + 'url' => 'test://images', + 'files' => ['1.jpg'], + ])->assertSuccessful(); + + $volume = $this->project()->volumes()->first(); + $this->assertEquals(ImageCsvParser::class, $volume->metadata_parser); + $this->assertTrue($volume->hasMetadata()); + $metaDisk->assertExists($volume->metadata_file_path); + $pendingMetaDisk->assertMissing($pv->metadata_file_path); + } + + public function testUpdateImportAnnotations() + { + $pendingMetaDisk = Storage::fake('pending-metadata'); + $fileDisk = Storage::fake('test'); + config(['volumes.editor_storage_disks' => ['test']]); + + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + 'metadata_file_path' => 'mymeta.csv', + 'metadata_parser' => ImageCsvParser::class, + ]); + $id = $pv->id; + $pendingMetaDisk->put('mymeta.csv', 'abc'); + + $fileDisk->makeDirectory('images'); + $fileDisk->put('images/1.jpg', 'abc'); + + $this->beAdmin(); + $this->putJson("/api/v1/pending-volumes/{$id}", [ + 'name' => 'my volume no. 1', + 'url' => 'test://images', + 'files' => ['1.jpg'], + 'import_annotations' => true, + ])->assertSuccessful(); + + $pv = $pv->fresh(); + $this->assertNotNull($pv); + $volume = $this->project()->volumes()->first(); + $this->assertEquals($volume->id, $pv->volume_id); + $this->assertTrue($pv->import_annotations); + } + + public function testUpdateImportFileLabels() + { + $pendingMetaDisk = Storage::fake('pending-metadata'); + $fileDisk = Storage::fake('test'); + config(['volumes.editor_storage_disks' => ['test']]); + + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + 'metadata_file_path' => 'mymeta.csv', + 'metadata_parser' => ImageCsvParser::class, + ]); + $id = $pv->id; + $pendingMetaDisk->put('mymeta.csv', 'abc'); + + $fileDisk->makeDirectory('images'); + $fileDisk->put('images/1.jpg', 'abc'); + + $this->beAdmin(); + $this->putJson("/api/v1/pending-volumes/{$id}", [ + 'name' => 'my volume no. 1', + 'url' => 'test://images', + 'files' => ['1.jpg'], + 'import_file_labels' => true, + ])->assertSuccessful(); + + $pv = $pv->fresh(); + $this->assertNotNull($pv); + $volume = $this->project()->volumes()->first(); + $this->assertEquals($volume->id, $pv->volume_id); + $this->assertTrue($pv->import_file_labels); + } + + public function testUpdateFromUIWithoutImport() + { + $disk = Storage::fake('test'); + config(['volumes.editor_storage_disks' => ['test']]); + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + ]); + $id = $pv->id; + + $disk->makeDirectory('images'); + $disk->put('images/1.jpg', 'abc'); + + $this->beAdmin(); + $response = $this->put("/api/v1/pending-volumes/{$id}", [ + 'name' => 'my volume no. 1', + 'url' => 'test://images', + 'files' => ['1.jpg'], + ]); + $volume = Volume::first(); + + $response->assertRedirectToRoute('volume', $volume->id); + } + + public function testUpdateFromUIWithAnnotationImport() + { + $disk = Storage::fake('test'); + config(['volumes.editor_storage_disks' => ['test']]); + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + ]); + $id = $pv->id; + + $disk->makeDirectory('images'); + $disk->put('images/1.jpg', 'abc'); + + $this->beAdmin(); + $response = $this->put("/api/v1/pending-volumes/{$id}", [ + 'name' => 'my volume no. 1', + 'url' => 'test://images', + 'import_annotations' => true, + 'files' => ['1.jpg'], + ]); + $volume = Volume::first(); + + $response->assertRedirectToRoute('pending-volume-annotation-labels', $id); + } + + public function testUpdateFromUIWithFileLabelImport() + { + $disk = Storage::fake('test'); + config(['volumes.editor_storage_disks' => ['test']]); + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + ]); + $id = $pv->id; + + $disk->makeDirectory('images'); + $disk->put('images/1.jpg', 'abc'); + + $this->beAdmin(); + $response = $this->put("/api/v1/pending-volumes/{$id}", [ + 'name' => 'my volume no. 1', + 'url' => 'test://images', + 'import_file_labels' => true, + 'files' => ['1.jpg'], + ]); + $volume = Volume::first(); + + $response->assertRedirectToRoute('pending-volume-file-labels', $id); + } + + public function testUpdateFileString() + { + $disk = Storage::fake('test'); + config(['volumes.editor_storage_disks' => ['test']]); + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + ]); + $id = $pv->id; + + $disk->makeDirectory('images'); + $disk->put('images/1.jpg', 'abc'); + $disk->put('images/2.jpg', 'abc'); + + $this->beAdmin(); + $this->putJson("/api/v1/pending-volumes/{$pv->id}", [ + 'name' => 'my volume no. 1', + 'url' => 'test://images', + 'files' => '"1.jpg" , 2.jpg , ,', + ])->assertSuccessful(); + + $volume = Volume::first(); + Queue::assertPushed(CreateNewImagesOrVideos::class, function ($job) { + $this->assertContains('1.jpg', $job->filenames); + $this->assertContains('2.jpg', $job->filenames); + $this->assertCount(2, $job->filenames); + + return true; + }); + } + + public function testUpdateHandle() + { + config(['volumes.editor_storage_disks' => ['test']]); + $disk = Storage::fake('test'); + $disk->makeDirectory('images'); + $disk->put('images/1.jpg', 'abc'); + + $id = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + ])->id; + + $this->beAdmin(); + // Invalid handle format. + $this->putJson("/api/v1/pending-volumes/{$id}", [ + 'name' => 'my volume no. 1', + 'url' => 'test://images', + 'files' => ['1.jpg'], + 'handle' => 'https://doi.org/10.3389/fmars.2017.00083', + ])->assertStatus(422); + + $this->putJson("/api/v1/pending-volumes/{$id}", [ + 'name' => 'my volume no. 1', + 'url' => 'test://images', + 'files' => ['1.jpg'], + 'handle' => '10.3389/fmars.2017.00083', + ])->assertStatus(200); + + $volume = Volume::orderBy('id', 'desc')->first(); + $this->assertEquals('10.3389/fmars.2017.00083', $volume->handle); + + $id = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + ])->id; + + // Some DOIs can contain multiple slashes. + $this->putJson("/api/v1/pending-volumes/{$id}", [ + 'name' => 'my volume no. 1', + 'url' => 'test://images', + 'files' => ['1.jpg'], + 'handle' => '10.3389/fmars.2017/00083', + ])->assertStatus(200); + + $id = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + ])->id; + + $this->putJson("/api/v1/pending-volumes/{$id}", [ + 'name' => 'my volume no. 1', + 'url' => 'test://images', + 'files' => ['1.jpg'], + 'handle' => '', + ])->assertStatus(200); + + $volume = Volume::orderBy('id', 'desc')->first(); + $this->assertNull($volume->handle); + } + + public function testUpdateFilesExist() + { + config(['volumes.editor_storage_disks' => ['test']]); + $disk = Storage::fake('test'); + $disk->makeDirectory('images'); + $disk->put('images/1.jpg', 'abc'); + + $id = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + ])->id; + + $this->beAdmin(); + + $this->putJson("/api/v1/pending-volumes/{$id}", [ + 'name' => 'my volume no. 1', + 'url' => 'test://images', + 'files' => ['1.jpg', '2.jpg'], + ])->assertStatus(422); + + $disk->put('images/2.jpg', 'abc'); + + $this->putJson("/api/v1/pending-volumes/{$id}", [ + 'name' => 'my volume no. 1', + 'url' => 'test://images', + 'files' => ['1.jpg', '2.jpg'], + ])->assertSuccessful(); + } + + public function testUpdateUnableToParseUri() + { + config(['volumes.editor_storage_disks' => ['test']]); + $disk = Storage::fake('test'); + $disk->makeDirectory('images'); + $disk->put('images/1.jpg', 'abc'); + $disk->put('images/2.jpg', 'abc'); + + $id = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + ])->id; + $this->beAdmin(); + $this->putJson("/api/v1/pending-volumes/{$id}", [ + 'name' => 'my volume no. 1', + 'url' => 'https:///my/images', + 'files' => ['1.jpg', '2.jpg'], + ])->assertStatus(422); + } + + public function testUpdateFilesExistException() + { + config(['volumes.editor_storage_disks' => ['test']]); + $disk = Storage::fake('test'); + $disk->makeDirectory('images'); + $disk->put('images/1.jpg', 'abc'); + + $id = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + ])->id; + $this->beAdmin(); + FileCache::shouldReceive('exists')->andThrow(new Exception('Invalid MIME type.')); + + $response = $this->putJson("/api/v1/pending-volumes/{$id}", [ + 'name' => 'my volume no. 1', + 'url' => 'test://images', + 'files' => ['1.jpg'], + ])->assertStatus(422); + + $this->assertStringContainsString('Some files could not be accessed. Invalid MIME type.', $response->getContent()); + } + + public function testUpdateVideos() + { + config(['volumes.editor_storage_disks' => ['test']]); + $disk = Storage::fake('test'); + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::videoId(), + 'user_id' => $this->admin()->id, + ]); + $id = $pv->id; + + $this->beAdmin(); + // invalid url format + $this->putJson("/api/v1/pending-volumes/{$id}", [ + 'name' => 'my volume no. 1', + 'url' => 'test', + 'files' => ['1.mp4', '2.mp4'], + ])->assertStatus(422); + + // unknown storage disk + $this->putJson("/api/v1/pending-volumes/{$id}", [ + 'name' => 'my volume no. 1', + 'url' => 'random', + 'files' => ['1.mp4', '2.mp4'], + ])->assertStatus(422); + + // videos directory dows not exist in storage disk + $this->putJson("/api/v1/pending-volumes/{$id}", [ + 'name' => 'my volume no. 1', + 'url' => 'test://videos', + 'files' => ['1.mp4', '2.mp4'], + ])->assertStatus(422); + + $disk->makeDirectory('videos'); + $disk->put('videos/1.mp4', 'abc'); + $disk->put('videos/2.mp4', 'abc'); + + // error because of duplicate video + $this->putJson("/api/v1/pending-volumes/{$id}", [ + 'name' => 'my volume no. 1', + 'url' => 'test://videos', + 'files' => ['1.mp4', '1.mp4'], + ])->assertStatus(422); + + // error because of unsupported video format + $this->putJson("/api/v1/pending-volumes/{$id}", [ + 'name' => 'my volume no. 1', + 'url' => 'test://videos', + 'files' => ['1.avi'], + ])->assertStatus(422); + + // Video filename too long. + $this->putJson("/api/v1/pending-volumes/{$id}", [ + 'name' => 'my volume no. 1', + 'url' => 'test://videos', + 'files' => ['aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.mp4'], + ])->assertStatus(422); + + $response = $this->putJson("/api/v1/pending-volumes/{$id}", [ + 'name' => 'my volume no. 1', + 'url' => 'test://videos', + 'media_type' => 'video', + // Elements should be sanitized and empty elements should be discarded + 'files' => ['" 1.mp4"', '', '\'2.mp4\' ', '', ''], + ])->assertSuccessful(); + + $id = json_decode($response->getContent())->volume_id; + Queue::assertPushed(CreateNewImagesOrVideos::class, function ($job) use ($id) { + $this->assertEquals($id, $job->volume->id); + $this->assertContains('1.mp4', $job->filenames); + $this->assertContains('2.mp4', $job->filenames); + + return true; + }); + + $this->assertNull($pv->fresh()); + + $this->assertEquals(1, $this->project()->volumes()->count()); + $volume = $this->project()->volumes()->first(); + $this->assertEquals('my volume no. 1', $volume->name); + $this->assertEquals('test://videos', $volume->url); + $this->assertEquals(MediaType::videoId(), $volume->media_type_id); + } + + public function testUpdateVideosWithMetadata() + { + $pendingMetaDisk = Storage::fake('pending-metadata'); + $metaDisk = Storage::fake('metadata'); + $fileDisk = Storage::fake('test'); + config(['volumes.editor_storage_disks' => ['test']]); + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::videoId(), + 'user_id' => $this->admin()->id, + 'metadata_file_path' => 'mymeta.csv', + 'metadata_parser' => VideoCsvParser::class, + ]); + $id = $pv->id; + $pendingMetaDisk->put('mymeta.csv', 'abc'); + + $fileDisk->makeDirectory('videos'); + $fileDisk->put('videos/1.mp4', 'abc'); + + $this->beAdmin(); + $this->putJson("/api/v1/pending-volumes/{$id}", [ + 'name' => 'my volume no. 1', + 'url' => 'test://videos', + 'files' => ['1.mp4'], + ])->assertSuccessful(); + + $volume = $this->project()->volumes()->first(); + $this->assertTrue($volume->hasMetadata()); + $metaDisk->assertExists($volume->metadata_file_path); + $pendingMetaDisk->assertMissing($pv->metadata_file_path); + } + + public function testUpdateProviderDenylist() + { + $id = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + ])->id; + $this->beAdmin(); + $this->putJson("/api/v1/pending-volumes/{$id}", [ + 'name' => 'my volume no. 1', + 'url' => 'https://dropbox.com', + 'files' => ['1.jpg', '2.jpg'], + ])->assertStatus(422); + } + + public function testUpdateAuthorizeDisk() + { + config(['volumes.admin_storage_disks' => ['admin-test']]); + config(['volumes.editor_storage_disks' => ['editor-test']]); + + $adminDisk = Storage::fake('admin-test'); + $adminDisk->put('images/1.jpg', 'abc'); + + $editorDisk = Storage::fake('editor-test'); + $editorDisk->put('images/2.jpg', 'abc'); + + $id = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + ])->id; + $this->beAdmin(); + $this->putJson("/api/v1/pending-volumes/{$id}", [ + 'name' => 'name', + 'url' => 'admin-test://images', + 'files' => ['1.jpg'], + ])->assertStatus(422); + + $this->putJson("/api/v1/pending-volumes/{$id}", [ + 'name' => 'name', + 'url' => 'editor-test://images', + 'files' => ['2.jpg'], + ])->assertSuccessful(); + + $id = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + ])->id; + + $this->beGlobalAdmin(); + + $this->putJson("/api/v1/pending-volumes/{$id}", [ + 'name' => 'name', + 'url' => 'editor-test://images', + 'files' => ['2.jpg'], + ])->assertStatus(422); + + $this->putJson("/api/v1/pending-volumes/{$id}", [ + 'name' => 'name', + 'url' => 'admin-test://images', + 'files' => ['1.jpg'], + ])->assertSuccessful(); + } + + public function testDestroy() + { + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + ]); + + $this->beExpert(); + $this->deleteJson("/api/v1/pending-volumes/{$pv->id}")->assertStatus(403); + + $this->beAdmin(); + $this->deleteJson("/api/v1/pending-volumes/{$pv->id}")->assertStatus(200); + $this->assertNull($pv->fresh()); + } + + public function testDestroyFromUI() + { + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + ]); + + $this->beAdmin(); + $this + ->delete("/api/v1/pending-volumes/{$pv->id}") + ->assertRedirectToRoute('create-volume', ['project' => $pv->project_id]); + } +} diff --git a/tests/php/Http/Controllers/Api/PendingVolumeImportControllerTest.php b/tests/php/Http/Controllers/Api/PendingVolumeImportControllerTest.php new file mode 100644 index 000000000..b69b5a78b --- /dev/null +++ b/tests/php/Http/Controllers/Api/PendingVolumeImportControllerTest.php @@ -0,0 +1,1199 @@ +addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $annotation = new ImageAnnotation( + shape: Shape::point(), + points: [10, 10], + labels: [$lau], + ); + $file->addAnnotation($annotation); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + 'metadata_file_path' => 'mymeta.csv', + ]); + $id = $pv->id; + + $this->doTestApiRoute('PUT', "/api/v1/pending-volumes/{$id}/annotation-labels"); + + $this->beExpert(); + $this + ->putJson("/api/v1/pending-volumes/{$id}/annotation-labels") + ->assertStatus(403); + + $this->beAdmin(); + // Label list required. + $this + ->putJson("/api/v1/pending-volumes/{$id}/annotation-labels") + ->assertStatus(422); + + // Label list must be filled. + $this->putJson("/api/v1/pending-volumes/{$id}/annotation-labels", [ + 'labels' => [], + ])->assertStatus(422); + + // No volume attached yet. + $this->putJson("/api/v1/pending-volumes/{$id}/annotation-labels", [ + 'labels' => [123], + ])->assertStatus(422); + + // Label not in metadata. + $this->putJson("/api/v1/pending-volumes/{$id}/annotation-labels", [ + 'labels' => [456], + ])->assertStatus(422); + + $pv->update(['volume_id' => $this->volume()->id]); + + $this->putJson("/api/v1/pending-volumes/{$id}/annotation-labels", [ + 'labels' => [123], + ])->assertSuccessful(); + + $pv->refresh(); + $this->assertEquals([123], $pv->only_annotation_labels); + } + + public function testUpdateAnnotationLabelsRedirectToSelectFileLabels() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('1.jpg'); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $annotation = new ImageAnnotation( + shape: Shape::point(), + points: [10, 10], + labels: [$lau], + ); + $file->addAnnotation($annotation); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + 'metadata_file_path' => 'mymeta.csv', + 'volume_id' => $this->volume()->id, + 'import_file_labels' => true, + ]); + $id = $pv->id; + + $this->beAdmin(); + $this->put("/api/v1/pending-volumes/{$id}/annotation-labels", [ + 'labels' => [123], + ])->assertRedirectToRoute('pending-volume-file-labels', $id); + } + + public function testUpdateAnnotationLabelsRedirectToLabelMap() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('1.jpg'); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $annotation = new ImageAnnotation( + shape: Shape::point(), + points: [10, 10], + labels: [$lau], + ); + $file->addAnnotation($annotation); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + 'metadata_file_path' => 'mymeta.csv', + 'volume_id' => $this->volume()->id, + 'import_file_labels' => false, + ]); + $id = $pv->id; + + $this->beAdmin(); + $this->put("/api/v1/pending-volumes/{$id}/annotation-labels", [ + 'labels' => [123], + ])->assertRedirectToRoute('pending-volume-label-map', $id); + } + + public function testUpdateFileLabels() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('1.jpg'); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $file->addFileLabel($lau); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + 'metadata_file_path' => 'mymeta.csv', + ]); + $id = $pv->id; + + $this->doTestApiRoute('PUT', "/api/v1/pending-volumes/{$id}/file-labels"); + + $this->beExpert(); + $this + ->putJson("/api/v1/pending-volumes/{$id}/file-labels") + ->assertStatus(403); + + $this->beAdmin(); + // Label list required. + $this + ->putJson("/api/v1/pending-volumes/{$id}/file-labels") + ->assertStatus(422); + + // Label list must be filled. + $this->putJson("/api/v1/pending-volumes/{$id}/file-labels", [ + 'labels' => [], + ])->assertStatus(422); + + // No volume attached yet. + $this->putJson("/api/v1/pending-volumes/{$id}/file-labels", [ + 'labels' => [123], + ])->assertStatus(422); + + $pv->update(['volume_id' => $this->volume()->id]); + + // Label not in metadata. + $this->putJson("/api/v1/pending-volumes/{$id}/file-labels", [ + 'labels' => [456], + ])->assertStatus(422); + + $this->putJson("/api/v1/pending-volumes/{$id}/file-labels", [ + 'labels' => [123], + ])->assertSuccessful(); + + $pv->refresh(); + $this->assertEquals([123], $pv->only_file_labels); + } + + public function testUpdateFileLabelsRedirectToLabelMap() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('1.jpg'); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $file->addFileLabel($lau); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + 'metadata_file_path' => 'mymeta.csv', + 'volume_id' => $this->volume()->id, + ]); + $id = $pv->id; + + $this->beAdmin(); + $this->put("/api/v1/pending-volumes/{$id}/file-labels", [ + 'labels' => [123], + ])->assertRedirectToRoute('pending-volume-label-map', $id); + } + + public function testUpdateLabelMap() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('1.jpg'); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $annotation = new ImageAnnotation( + shape: Shape::point(), + points: [10, 10], + labels: [$lau], + ); + $file->addAnnotation($annotation); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + 'metadata_file_path' => 'mymeta.csv', + ]); + $id = $pv->id; + + $this->doTestApiRoute('PUT', "/api/v1/pending-volumes/{$id}/label-map"); + + $this->beExpert(); + $this + ->putJson("/api/v1/pending-volumes/{$id}/label-map") + ->assertStatus(403); + + $this->beAdmin(); + // Label map required. + $this + ->putJson("/api/v1/pending-volumes/{$id}/label-map") + ->assertStatus(422); + + // Label map must be filled. + $this->putJson("/api/v1/pending-volumes/{$id}/label-map", [ + 'label_map' => [], + ])->assertStatus(422); + + // Map values must be ints. + $this->putJson("/api/v1/pending-volumes/{$id}/label-map", [ + 'label_map' => ['abc', 'def'], + ])->assertStatus(422); + + // No volume attached yet. + $this->putJson("/api/v1/pending-volumes/{$id}/label-map", [ + 'label_map' => [123 => $this->labelRoot()->id], + ])->assertStatus(422); + + $pv->update(['volume_id' => $this->volume()->id]); + + // Label not in metadata. + $this->putJson("/api/v1/pending-volumes/{$id}/label-map", [ + 'label_map' => [456 => $this->labelRoot()->id], + ])->assertStatus(422); + + // Label not in database. + $this->putJson("/api/v1/pending-volumes/{$id}/label-map", [ + 'label_map' => [123 => -1], + ])->assertStatus(422); + + $this->putJson("/api/v1/pending-volumes/{$id}/label-map", [ + 'label_map' => [123 => "{$this->labelRoot()->id}"], + ])->assertSuccessful(); + + $pv->refresh(); + // Values should be cast to int. + $this->assertSame([123 => $this->labelRoot()->id], $pv->label_map); + } + + public function testUpdateLabelMapFileLabel() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('1.jpg'); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $file->addFileLabel($lau); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + 'metadata_file_path' => 'mymeta.csv', + 'volume_id' => $this->volume()->id, + ]); + $id = $pv->id; + + $this->beAdmin(); + + $this->putJson("/api/v1/pending-volumes/{$id}/label-map", [ + 'label_map' => [123 => $this->labelRoot()->id], + ])->assertSuccessful(); + + $pv->refresh(); + $this->assertEquals([123 => $this->labelRoot()->id], $pv->label_map); + } + + public function testUpdateLabelMapTryIgnoredAnnotationLabel() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('1.jpg'); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $annotation = new ImageAnnotation( + shape: Shape::point(), + points: [10, 10], + labels: [$lau], + ); + $file->addAnnotation($annotation); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + 'metadata_file_path' => 'mymeta.csv', + 'volume_id' => $this->volume()->id, + 'only_annotation_labels' => [1], + ]); + $id = $pv->id; + + $this->beAdmin(); + $this->putJson("/api/v1/pending-volumes/{$id}/label-map", [ + 'label_map' => [123 => $this->labelRoot()->id], + ])->assertStatus(422); + } + + public function testUpdateLabelMapTryIgnoredFileLabel() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('1.jpg'); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $file->addFileLabel($lau); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + 'metadata_file_path' => 'mymeta.csv', + 'volume_id' => $this->volume()->id, + 'only_file_labels' => [1], + ]); + $id = $pv->id; + + $this->beAdmin(); + $this->putJson("/api/v1/pending-volumes/{$id}/label-map", [ + 'label_map' => [123 => $this->labelRoot()->id], + ])->assertStatus(422); + } + + public function testUpdateLabelMapTryLabelNotAllowed() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('1.jpg'); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $file->addFileLabel($lau); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + 'metadata_file_path' => 'mymeta.csv', + 'volume_id' => $this->volume()->id, + ]); + $id = $pv->id; + + // This label belongs to a label tree that is not accessible by the user. + // The other tests use a label from a public label tree. + $dbLabel = DbLabel::factory()->create([ + 'label_tree_id' => LabelTree::factory()->create([ + 'visibility_id' => Visibility::privateId(), + ])->id, + ]); + + $this->beAdmin(); + $this->putJson("/api/v1/pending-volumes/{$id}/label-map", [ + 'label_map' => [123 => $dbLabel->id], + ])->assertStatus(422); + } + + public function testUpdateLabelMapTryLabelPrivate() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('1.jpg'); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $file->addFileLabel($lau); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + 'metadata_file_path' => 'mymeta.csv', + 'volume_id' => $this->volume()->id, + ]); + $id = $pv->id; + + $dbLabel = DbLabel::factory()->create([ + 'label_tree_id' => LabelTree::factory()->create([ + 'visibility_id' => Visibility::privateId(), + ])->id, + ]); + + $dbLabel->tree->addMember($this->admin(), Role::admin()); + + $this->beAdmin(); + $this->putJson("/api/v1/pending-volumes/{$id}/label-map", [ + 'label_map' => [123 => $dbLabel->id], + ])->assertStatus(200); + } + + public function testUpdateLabelMapRedirectToUserMap() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('1.jpg'); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $file->addFileLabel($lau); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + 'metadata_file_path' => 'mymeta.csv', + 'volume_id' => $this->volume()->id, + ]); + $id = $pv->id; + + $this->beAdmin(); + + $this->put("/api/v1/pending-volumes/{$id}/label-map", [ + 'label_map' => [123 => $this->labelRoot()->id], + ])->assertRedirectToRoute('pending-volume-user-map', $id); + } + + public function testUpdateUserMap() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('1.jpg'); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $annotation = new ImageAnnotation( + shape: Shape::point(), + points: [10, 10], + labels: [$lau], + ); + $file->addAnnotation($annotation); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + 'metadata_file_path' => 'mymeta.csv', + ]); + $id = $pv->id; + + $this->doTestApiRoute('PUT', "/api/v1/pending-volumes/{$id}/user-map"); + + $this->beExpert(); + $this + ->putJson("/api/v1/pending-volumes/{$id}/user-map") + ->assertStatus(403); + + $this->beAdmin(); + // User map required. + $this + ->putJson("/api/v1/pending-volumes/{$id}/user-map") + ->assertStatus(422); + + // User map must be filled. + $this->putJson("/api/v1/pending-volumes/{$id}/user-map", [ + 'user_map' => [], + ])->assertStatus(422); + + // No volume attached yet. + $this->putJson("/api/v1/pending-volumes/{$id}/user-map", [ + 'user_map' => [321 => $this->user()->id], + ])->assertStatus(422); + + $pv->update(['volume_id' => $this->volume()->id]); + + // User not in metadata. + $this->putJson("/api/v1/pending-volumes/{$id}/user-map", [ + 'user_map' => [456 => $this->user()->id], + ])->assertStatus(422); + + // User not in database. + $this->putJson("/api/v1/pending-volumes/{$id}/user-map", [ + 'user_map' => [321 => -1], + ])->assertStatus(422); + + $this->putJson("/api/v1/pending-volumes/{$id}/user-map", [ + 'user_map' => [321 => "{$this->user()->id}"], + ])->assertSuccessful(); + + $pv->refresh(); + $this->assertSame([321 => $this->user()->id], $pv->user_map); + } + + public function testUpdateUserMapFileLabel() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('1.jpg'); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $file->addFileLabel($lau); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + 'metadata_file_path' => 'mymeta.csv', + 'volume_id' => $this->volume()->id, + ]); + $id = $pv->id; + + $this->beAdmin(); + + $this->putJson("/api/v1/pending-volumes/{$id}/user-map", [ + 'user_map' => [321 => $this->user()->id], + ])->assertSuccessful(); + + $pv->refresh(); + $this->assertEquals([321 => $this->user()->id], $pv->user_map); + } + + public function testUpdateUserMapRedirectToFinish() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('1.jpg'); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $file->addFileLabel($lau); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + 'metadata_file_path' => 'mymeta.csv', + 'volume_id' => $this->volume()->id, + ]); + $id = $pv->id; + + $this->beAdmin(); + + $this->put("/api/v1/pending-volumes/{$id}/user-map", [ + 'user_map' => [321 => $this->user()->id], + ])->assertRedirectToRoute('pending-volume-finish', $id); + } + + public function testStoreImport() + { + $dbUser = DbUser::factory()->create(); + $dbLabel = DbLabel::factory()->create(); + $metadata = new VolumeMetadata; + $file = new ImageMetadata('1.jpg'); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $file->addFileLabel($lau); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + 'metadata_file_path' => 'mymeta.csv', + 'import_file_labels' => true, + 'user_map' => [321 => $dbUser->id], + 'label_map' => [123 => $dbLabel->id], + ]); + $id = $pv->id; + + $this->doTestApiRoute('POST', "/api/v1/pending-volumes/{$id}/import"); + + $this->beExpert(); + $this->postJson("/api/v1/pending-volumes/{$id}/import")->assertStatus(403); + + $this->beAdmin(); + + // No volume_id set. + $this->postJson("/api/v1/pending-volumes/{$id}/import")->assertStatus(422); + + $pv->update(['volume_id' => $this->volume()->id]); + + $this->postJson("/api/v1/pending-volumes/{$id}/import")->assertSuccessful(); + + Queue::assertPushed(ImportVolumeMetadata::class, function ($job) use ($pv) { + $this->assertEquals($pv->id, $job->pv->id); + + return true; + }); + + $this->assertTrue($pv->fresh()->importing); + } + + public function testStoreImportNoImportAnnotationsSet() + { + $dbUser = DbUser::factory()->create(); + $dbLabel = DbLabel::factory()->create(); + $metadata = new VolumeMetadata; + $file = new ImageMetadata('1.jpg'); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $annotation = new ImageAnnotation( + shape: Shape::point(), + points: [10, 10], + labels: [$lau], + ); + $file->addAnnotation($annotation); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + 'metadata_file_path' => 'mymeta.csv', + 'volume_id' => $this->volume()->id, + 'user_map' => [321 => $dbUser->id], + 'label_map' => [123 => $dbLabel->id], + ]); + $id = $pv->id; + + $this->beAdmin(); + $this->postJson("/api/v1/pending-volumes/{$id}/import")->assertStatus(422); + $pv->update(['import_annotations' => true]); + $this->postJson("/api/v1/pending-volumes/{$id}/import")->assertSuccessful(); + } + + public function testStoreImportNoImportFileLabelsSet() + { + $dbUser = DbUser::factory()->create(); + $dbLabel = DbLabel::factory()->create(); + $metadata = new VolumeMetadata; + $file = new ImageMetadata('1.jpg'); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $file->addFileLabel($lau); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + 'metadata_file_path' => 'mymeta.csv', + 'volume_id' => $this->volume()->id, + 'user_map' => [321 => $dbUser->id], + 'label_map' => [123 => $dbLabel->id], + ]); + $id = $pv->id; + + $this->beAdmin(); + $this->postJson("/api/v1/pending-volumes/{$id}/import")->assertStatus(422); + $pv->update(['import_file_labels' => true]); + $this->postJson("/api/v1/pending-volumes/{$id}/import")->assertSuccessful(); + } + + public function testStoreImportNoAnnotations() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('1.jpg'); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $file->addFileLabel($lau); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + 'metadata_file_path' => 'mymeta.csv', + 'volume_id' => $this->volume()->id, + 'import_annotations' => true, + ]); + $id = $pv->id; + + $this->beAdmin(); + $this->postJson("/api/v1/pending-volumes/{$id}/import")->assertStatus(422); + } + + public function testStoreImportNoAnnotationsAfterFiltering() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('1.jpg'); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $annotation = new ImageAnnotation( + shape: Shape::point(), + points: [10, 10], + labels: [$lau], + ); + $file->addAnnotation($annotation); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + 'metadata_file_path' => 'mymeta.csv', + 'volume_id' => $this->volume()->id, + 'import_annotations' => true, + 'only_annotation_labels' => [1], + ]); + $id = $pv->id; + + $this->beAdmin(); + $this->postJson("/api/v1/pending-volumes/{$id}/import")->assertStatus(422); + } + + public function testStoreImportNoFileLabels() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('1.jpg'); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $annotation = new ImageAnnotation( + shape: Shape::point(), + points: [10, 10], + labels: [$lau], + ); + $file->addAnnotation($annotation); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + 'metadata_file_path' => 'mymeta.csv', + 'volume_id' => $this->volume()->id, + 'import_file_labels' => true, + ]); + $id = $pv->id; + + $this->beAdmin(); + $this->postJson("/api/v1/pending-volumes/{$id}/import")->assertStatus(422); + } + + public function testStoreImportNoFileLabelsAfterFiltering() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('1.jpg'); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $file->addFileLabel($lau); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + 'metadata_file_path' => 'mymeta.csv', + 'volume_id' => $this->volume()->id, + 'import_file_labels' => true, + 'only_file_labels' => [1], + ]); + $id = $pv->id; + + $this->beAdmin(); + $this->postJson("/api/v1/pending-volumes/{$id}/import")->assertStatus(422); + } + + public function testStoreImportAlreadyImporting() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('1.jpg'); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $file->addFileLabel($lau); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + 'metadata_file_path' => 'mymeta.csv', + 'volume_id' => $this->volume()->id, + 'import_file_labels' => true, + 'label_map' => [123 => $this->labelRoot()->id], + 'user_map' => [321 => $this->user()->id], + 'importing' => true, + ]); + $id = $pv->id; + + $this->beAdmin(); + $this->postJson("/api/v1/pending-volumes/{$id}/import")->assertStatus(422); + } + + public function testStoreImportMatchByUuid() + { + $dbUser = DbUser::factory()->create(); + $dbLabel = DbLabel::factory()->create(); + $metadata = new VolumeMetadata; + $file = new ImageMetadata('1.jpg'); + $metadata->addFile($file); + $label = new Label(123, 'my label', uuid: $dbLabel->uuid); + $user = new User(321, 'joe user', uuid: $dbUser->uuid); + $lau = new LabelAndUser($label, $user); + $file->addFileLabel($lau); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + 'metadata_file_path' => 'mymeta.csv', + 'volume_id' => $this->volume()->id, + 'import_file_labels' => true, + ]); + $id = $pv->id; + + $this->beAdmin(); + $this->postJson("/api/v1/pending-volumes/{$id}/import")->assertSuccessful(); + } + + public function testStoreImportUserNoMatch() + { + $dbLabel = DbLabel::factory()->create(); + $metadata = new VolumeMetadata; + $file = new ImageMetadata('1.jpg'); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $file->addFileLabel($lau); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + 'metadata_file_path' => 'mymeta.csv', + 'volume_id' => $this->volume()->id, + 'import_file_labels' => true, + 'label_map' => [123 => $dbLabel->id], + ]); + $id = $pv->id; + + $this->beAdmin(); + $this->postJson("/api/v1/pending-volumes/{$id}/import")->assertStatus(422); + } + + public function testStoreImportLabelNoMatch() + { + $dbUser = DbUser::factory()->create(); + $metadata = new VolumeMetadata; + $file = new ImageMetadata('1.jpg'); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $file->addFileLabel($lau); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + 'metadata_file_path' => 'mymeta.csv', + 'volume_id' => $this->volume()->id, + 'import_file_labels' => true, + 'user_map' => [321 => $dbUser->id], + ]); + $id = $pv->id; + + $this->beAdmin(); + $this->postJson("/api/v1/pending-volumes/{$id}/import")->assertStatus(422); + } + + public function testStoreImportMatchOnlyWithAnnotationLabelFilter() + { + $dbUser = DbUser::factory()->create(); + $dbLabel = DbLabel::factory()->create(); + $metadata = new VolumeMetadata; + $file = new ImageMetadata('1.jpg'); + $metadata->addFile($file); + $label1 = new Label(123, 'my label'); + $user1 = new User(321, 'joe user'); + $lau = new LabelAndUser($label1, $user1); + $annotation = new ImageAnnotation( + shape: Shape::point(), + points: [10, 10], + labels: [$lau], + ); + $file->addAnnotation($annotation); + $label2 = new Label(456, 'my label'); + $user2 = new User(654, 'joe user'); + $lau = new LabelAndUser($label2, $user2); + $annotation = new ImageAnnotation( + shape: Shape::point(), + points: [10, 10], + labels: [$lau], + ); + $file->addAnnotation($annotation); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + 'metadata_file_path' => 'mymeta.csv', + 'volume_id' => $this->volume()->id, + 'import_annotations' => true, + 'only_annotation_labels' => [123], + 'user_map' => [321 => $dbUser->id], + 'label_map' => [123 => $dbLabel->id], + + ]); + $id = $pv->id; + + $this->beAdmin(); + $this->postJson("/api/v1/pending-volumes/{$id}/import")->assertSuccessful(); + } + + public function testStoreImportMatchOnlyWithFileLabelFilter() + { + $dbUser = DbUser::factory()->create(); + $dbLabel = DbLabel::factory()->create(); + $metadata = new VolumeMetadata; + $file = new ImageMetadata('1.jpg'); + $metadata->addFile($file); + $label1 = new Label(123, 'my label'); + $user1 = new User(321, 'joe user'); + $lau = new LabelAndUser($label1, $user1); + $file->addFileLabel($lau); + $label2 = new Label(456, 'my label'); + $user2 = new User(654, 'joe user'); + $lau = new LabelAndUser($label2, $user2); + $file->addFileLabel($lau); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + 'metadata_file_path' => 'mymeta.csv', + 'volume_id' => $this->volume()->id, + 'import_file_labels' => true, + 'only_file_labels' => [123], + 'user_map' => [321 => $dbUser->id], + 'label_map' => [123 => $dbLabel->id], + + ]); + $id = $pv->id; + + $this->beAdmin(); + $this->postJson("/api/v1/pending-volumes/{$id}/import")->assertSuccessful(); + } + + public function testStoreImportValidateImageAnnotationPoints() + { + $dbUser = DbUser::factory()->create(); + $dbLabel = DbLabel::factory()->create(); + $metadata = new VolumeMetadata; + $file = new ImageMetadata('1.jpg'); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $annotation = new ImageAnnotation( + shape: Shape::point(), + // Incorrect points for the shape. + points: [10, 10, 10], + labels: [$lau], + ); + $file->addAnnotation($annotation); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + 'metadata_file_path' => 'mymeta.csv', + 'volume_id' => $this->volume()->id, + 'user_map' => [321 => $dbUser->id], + 'label_map' => [123 => $dbLabel->id], + 'import_annotations' => true, + ]); + $id = $pv->id; + + $this->beAdmin(); + $this->postJson("/api/v1/pending-volumes/{$id}/import")->assertStatus(422); + } + + public function testStoreImportValidateVideoAnnotationPoints() + { + $dbUser = DbUser::factory()->create(); + $dbLabel = DbLabel::factory()->create(); + $metadata = new VolumeMetadata; + $file = new VideoMetadata('1.jpg'); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $annotation = new VideoAnnotation( + shape: Shape::point(), + // Must be an array of arrays. + points: [10, 10], + frames: [1], + labels: [$lau], + ); + $file->addAnnotation($annotation); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::videoId(), + 'user_id' => $this->admin()->id, + 'metadata_file_path' => 'mymeta.csv', + 'volume_id' => $this->volume()->id, + 'user_map' => [321 => $dbUser->id], + 'label_map' => [123 => $dbLabel->id], + 'import_annotations' => true, + ]); + $id = $pv->id; + + $this->beAdmin(); + $this->postJson("/api/v1/pending-volumes/{$id}/import")->assertStatus(422); + } + + public function testStoreImportValidateVideoAnnotationFrames() + { + $dbUser = DbUser::factory()->create(); + $dbLabel = DbLabel::factory()->create(); + $metadata = new VolumeMetadata; + $file = new VideoMetadata('1.jpg'); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $annotation = new VideoAnnotation( + shape: Shape::point(), + points: [[10, 10]], + // Must have the same number of elements than points. + frames: [], + labels: [$lau], + ); + $file->addAnnotation($annotation); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::videoId(), + 'user_id' => $this->admin()->id, + 'metadata_file_path' => 'mymeta.csv', + 'volume_id' => $this->volume()->id, + 'user_map' => [321 => $dbUser->id], + 'label_map' => [123 => $dbLabel->id], + 'import_annotations' => true, + ]); + $id = $pv->id; + + $this->beAdmin(); + $this->postJson("/api/v1/pending-volumes/{$id}/import")->assertStatus(422); + } + + public function testStoreImportRedirectToVolume() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('1.jpg'); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $file->addFileLabel($lau); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'media_type_id' => MediaType::imageId(), + 'user_id' => $this->admin()->id, + 'metadata_file_path' => 'mymeta.csv', + 'volume_id' => $this->volume()->id, + 'import_file_labels' => true, + 'label_map' => [123 => $this->labelRoot()->id], + 'user_map' => [321 => $this->user()->id], + ]); + $id = $pv->id; + + $this->beAdmin(); + $this + ->post("/api/v1/pending-volumes/{$id}/import") + ->assertRedirectToRoute('volume', $this->volume()->id); + } +} diff --git a/tests/php/Http/Controllers/Api/ProjectVolumeControllerTest.php b/tests/php/Http/Controllers/Api/ProjectVolumeControllerTest.php index 75ac8a3eb..15fb95c5c 100644 --- a/tests/php/Http/Controllers/Api/ProjectVolumeControllerTest.php +++ b/tests/php/Http/Controllers/Api/ProjectVolumeControllerTest.php @@ -7,6 +7,8 @@ use Biigle\Jobs\CreateNewImagesOrVideos; use Biigle\Jobs\DeleteVolume; use Biigle\Role; +use Biigle\Services\MetadataParsing\ImageCsvParser; +use Biigle\Services\MetadataParsing\VideoCsvParser; use Biigle\Tests\ProjectTest; use Biigle\Tests\VolumeTest; use Biigle\Video; @@ -446,11 +448,14 @@ public function testStoreEmptyImageMetadataText() ]) ->assertSuccessful(); - Queue::assertPushed(CreateNewImagesOrVideos::class, fn ($job) => empty($job->metadata)); + $volume = $this->project()->volumes()->first(); + $this->assertNull($volume->metadata_file_path); + $this->assertNull($volume->metadata_parser); } public function testStoreImageMetadataText() { + Storage::fake('metadata'); $id = $this->project()->id; $this->beAdmin(); Storage::disk('test')->makeDirectory('images'); @@ -466,11 +471,15 @@ public function testStoreImageMetadataText() ]) ->assertSuccessful(); - Queue::assertPushed(CreateNewImagesOrVideos::class, fn ($job) => $job->metadata[1][0] === '1.jpg' && $job->metadata[1][1] === '2.5'); + $volume = Volume::orderBy('id', 'desc')->first(); + $this->assertNotNull($volume->metadata_file_path); + $this->assertEquals(ImageCsvParser::class, $volume->metadata_parser); } public function testStoreImageMetadataCsv() { + Storage::fake('metadata'); + $id = $this->project()->id; $this->beAdmin(); $csv = __DIR__."/../../../../files/image-metadata.csv"; @@ -478,17 +487,17 @@ public function testStoreImageMetadataCsv() Storage::disk('test')->makeDirectory('images'); Storage::disk('test')->put('images/abc.jpg', 'abc'); - $this - ->postJson("/api/v1/projects/{$id}/volumes", [ - 'name' => 'my volume no. 1', - 'url' => 'test://images', - 'media_type' => 'image', - 'files' => 'abc.jpg', - 'metadata_csv' => $file, - ]) - ->assertSuccessful(); + $this->postJson("/api/v1/projects/{$id}/volumes", [ + 'name' => 'my volume no. 1', + 'url' => 'test://images', + 'media_type' => 'image', + 'files' => 'abc.jpg', + 'metadata_csv' => $file, + ])->assertSuccessful(); - Queue::assertPushed(CreateNewImagesOrVideos::class, fn ($job) => $job->metadata[1][0] === 'abc.jpg' && $job->metadata[1][6] === '2.6'); + $volume = Volume::orderBy('id', 'desc')->first(); + $this->assertNotNull($volume->metadata_file_path); + $this->assertEquals(ImageCsvParser::class, $volume->metadata_parser); } public function testStoreImageMetadataInvalid() @@ -504,7 +513,7 @@ public function testStoreImageMetadataInvalid() 'url' => 'test://images', 'media_type' => 'image', 'files' => '1.jpg', - 'metadata_text' => "filename,area\nabc.jpg,2.5", + 'metadata_text' => "filename,yaw\nabc.jpg,400", ]) ->assertStatus(422); } @@ -526,11 +535,14 @@ public function testStoreEmptyVideoMetadataText() ]) ->assertSuccessful(); - Queue::assertPushed(CreateNewImagesOrVideos::class, fn ($job) => empty($job->metadata)); + $volume = $this->project()->volumes()->first(); + $this->assertNull($volume->metadata_file_path); + $this->assertNull($volume->metadata_parser); } public function testStoreVideoMetadataText() { + Storage::fake('metadata'); $id = $this->project()->id; $this->beAdmin(); Storage::disk('test')->makeDirectory('videos'); @@ -546,11 +558,15 @@ public function testStoreVideoMetadataText() ]) ->assertSuccessful(); - Queue::assertPushed(CreateNewImagesOrVideos::class, fn ($job) => $job->metadata[1][0] === '1.mp4' && $job->metadata[1][1] === '2.5'); + $volume = Volume::orderBy('id', 'desc')->first(); + $this->assertNotNull($volume->metadata_file_path); + $this->assertEquals(VideoCsvParser::class, $volume->metadata_parser); } public function testStoreVideoMetadataCsv() { + Storage::fake('metadata'); + $id = $this->project()->id; $this->beAdmin(); $csv = __DIR__."/../../../../files/video-metadata.csv"; @@ -568,7 +584,9 @@ public function testStoreVideoMetadataCsv() ]) ->assertSuccessful(); - Queue::assertPushed(CreateNewImagesOrVideos::class, fn ($job) => $job->metadata[1][0] === 'abc.mp4' && $job->metadata[1][6] === '2.6' && $job->metadata[2][6] === '1.6'); + $volume = Volume::orderBy('id', 'desc')->first(); + $this->assertNotNull($volume->metadata_file_path); + $this->assertEquals(VideoCsvParser::class, $volume->metadata_parser); } public function testStoreVideoMetadataInvalid() @@ -584,97 +602,7 @@ public function testStoreVideoMetadataInvalid() 'url' => 'test://videos', 'media_type' => 'video', 'files' => '1.mp4', - 'metadata_text' => "filename,area\nabc.mp4,2.5", - ]) - ->assertStatus(422); - } - - public function testStoreImageIfdoFile() - { - $id = $this->project()->id; - $this->beAdmin(); - $csv = __DIR__."/../../../../files/image-ifdo.yaml"; - $file = new UploadedFile($csv, 'ifdo.yaml', 'application/yaml', null, true); - Storage::disk('test')->makeDirectory('images'); - Storage::disk('test')->put('images/abc.jpg', 'abc'); - - Storage::fake('ifdos'); - - $this - ->postJson("/api/v1/projects/{$id}/volumes", [ - 'name' => 'my volume no. 1', - 'url' => 'test://images', - 'media_type' => 'image', - 'files' => 'abc.jpg', - 'ifdo_file' => $file, - ]) - ->assertSuccessful(); - - $volume = Volume::orderBy('id', 'desc')->first(); - $this->assertTrue($volume->hasIfdo()); - } - - public function testStoreVideoIfdoFile() - { - $id = $this->project()->id; - $this->beAdmin(); - $csv = __DIR__."/../../../../files/video-ifdo.yaml"; - $file = new UploadedFile($csv, 'ifdo.yaml', 'application/yaml', null, true); - Storage::disk('test')->makeDirectory('videos'); - Storage::disk('test')->put('videos/abc.mp4', 'abc'); - - Storage::fake('ifdos'); - - $this - ->postJson("/api/v1/projects/{$id}/volumes", [ - 'name' => 'my volume no. 1', - 'url' => 'test://videos', - 'media_type' => 'video', - 'files' => 'abc.mp4', - 'ifdo_file' => $file, - ]) - ->assertSuccessful(); - - $volume = Volume::orderBy('id', 'desc')->first(); - $this->assertTrue($volume->hasIfdo()); - } - - public function testStoreVideoVolumeWithImageIfdoFile() - { - $id = $this->project()->id; - $this->beAdmin(); - $csv = __DIR__."/../../../../files/image-ifdo.yaml"; - $file = new UploadedFile($csv, 'ifdo.yaml', 'application/yaml', null, true); - Storage::disk('test')->makeDirectory('videos'); - Storage::disk('test')->put('videos/abc.mp4', 'abc'); - - $this - ->postJson("/api/v1/projects/{$id}/volumes", [ - 'name' => 'my volume no. 1', - 'url' => 'test://videos', - 'media_type' => 'video', - 'files' => 'abc.mp4', - 'ifdo_file' => $file, - ]) - ->assertStatus(422); - } - - public function testStoreImageVolumeWithVideoIfdoFile() - { - $id = $this->project()->id; - $this->beAdmin(); - $csv = __DIR__."/../../../../files/video-ifdo.yaml"; - $file = new UploadedFile($csv, 'ifdo.yaml', 'application/yaml', null, true); - Storage::disk('test')->makeDirectory('images'); - Storage::disk('test')->put('images/abc.jpg', 'abc'); - - $this - ->postJson("/api/v1/projects/{$id}/volumes", [ - 'name' => 'my volume no. 1', - 'url' => 'test://images', - 'media_type' => 'image', - 'files' => 'abc.jpg', - 'ifdo_file' => $file, + 'metadata_text' => "filename,yaw\nabc.mp4,400", ]) ->assertStatus(422); } @@ -750,7 +678,6 @@ public function testAttach() $secondProject = ProjectTest::create(); $pid = $secondProject->id; - // $secondProject->addUserId($this->admin()->id, Role::adminId()); $this->doTestApiRoute('POST', "/api/v1/projects/{$pid}/volumes/{$tid}"); diff --git a/tests/php/Http/Controllers/Api/VolumeControllerTest.php b/tests/php/Http/Controllers/Api/VolumeControllerTest.php index a65711e59..7896274db 100644 --- a/tests/php/Http/Controllers/Api/VolumeControllerTest.php +++ b/tests/php/Http/Controllers/Api/VolumeControllerTest.php @@ -263,12 +263,7 @@ public function testUpdateRedirect() public function testCloneVolume() { - $volume = $this - ->volume([ - 'created_at' => '2022-11-09 14:37:00', - 'updated_at' => '2022-11-09 14:37:00', - ]) - ->fresh(); + $volume = $this->volume(['metadata_file_path' => 'mymeta.csv']); $project = ProjectTest::create(); $this->doTestApiRoute('POST', "/api/v1/volumes/{$volume->id}/clone-to/{$project->id}"); @@ -289,21 +284,34 @@ public function testCloneVolume() Cache::flush(); - Queue::fake(); - $this ->postJson("/api/v1/volumes/{$volume->id}/clone-to/{$project->id}") ->assertStatus(201); Queue::assertPushed(CloneImagesOrVideos::class); - // The target project. + $this->assertTrue($project->volumes()->exists()); + $copy = $project->volumes()->first(); + $this->assertEquals($copy->name, $this->volume()->name); + $this->assertEquals($copy->media_type_id, $this->volume()->media_type_id); + $this->assertEquals($copy->url, $this->volume()->url); + $this->assertTrue($copy->creating_async); + $this->assertEquals("{$copy->id}.csv", $copy->metadata_file_path); + } + + public function testCloneVolumeNewName() + { + $volume = $this->volume(['name' => 'myvolume']); $project = ProjectTest::create(); + $project->addUserId($this->admin()->id, Role::adminId()); $this->beAdmin(); - $project->addUserId($this->admin()->id, Role::adminId()); - $response = $this->postJson("/api/v1/volumes/{$volume->id}/clone-to/{$project->id}"); - $response->assertStatus(201); - Queue::assertPushed(CloneImagesOrVideos::class); + $this + ->postJson("/api/v1/volumes/{$volume->id}/clone-to/{$project->id}", [ + 'name' => 'volumecopy', + ]) + ->assertStatus(201); + $copy = $project->volumes()->first(); + $this->assertEquals($copy->name, 'volumecopy'); } } diff --git a/tests/php/Http/Controllers/Api/Volumes/IfdoControllerTest.php b/tests/php/Http/Controllers/Api/Volumes/IfdoControllerTest.php deleted file mode 100644 index f78817d5c..000000000 --- a/tests/php/Http/Controllers/Api/Volumes/IfdoControllerTest.php +++ /dev/null @@ -1,54 +0,0 @@ -volume()->id; - - $this->doTestApiRoute('GET', "/api/v1/volumes/{$id}/ifdo"); - - $this->beUser(); - $this->getJson("/api/v1/volumes/{$id}/ifdo") - ->assertStatus(403); - - $this->beGuest(); - $this->getJson("/api/v1/volumes/{$id}/ifdo") - ->assertStatus(404); - - $disk = Storage::fake('ifdos'); - $disk->put($id.'.yaml', 'abc'); - - $this->getJson("/api/v1/volumes/-1/ifdo") - ->assertStatus(404); - - $response = $this->getJson("/api/v1/volumes/{$id}/ifdo"); - $response->assertStatus(200); - $this->assertSame("attachment; filename=biigle-volume-{$id}-ifdo.yaml", $response->headers->get('content-disposition')); - } - - public function testDestroy() - { - $id = $this->volume()->id; - - $this->doTestApiRoute('DELETE', "/api/v1/volumes/{$id}/ifdo"); - - $disk = Storage::fake('ifdos'); - $disk->put($id.'.yaml', 'abc'); - - $this->beExpert(); - $this->deleteJson("/api/v1/volumes/{$id}/ifdo") - ->assertStatus(403); - - $this->beAdmin(); - $this->deleteJson("/api/v1/volumes/{$id}/ifdo") - ->assertSuccessful(); - - $this->assertFalse($this->volume()->hasIfdo()); - } -} diff --git a/tests/php/Http/Controllers/Api/Volumes/MetadataControllerTest.php b/tests/php/Http/Controllers/Api/Volumes/MetadataControllerTest.php index a10067672..a8e9c44a2 100644 --- a/tests/php/Http/Controllers/Api/Volumes/MetadataControllerTest.php +++ b/tests/php/Http/Controllers/Api/Volumes/MetadataControllerTest.php @@ -3,448 +3,176 @@ namespace Biigle\Tests\Http\Controllers\Api\Volumes; use ApiTestCase; +use Biigle\Jobs\UpdateVolumeMetadata; use Biigle\MediaType; -use Biigle\Tests\ImageTest; -use Biigle\Tests\VideoTest; +use Biigle\Services\MetadataParsing\ImageCsvParser; +use Biigle\Services\MetadataParsing\VideoCsvParser; use Illuminate\Http\UploadedFile; +use Queue; use Storage; class MetadataControllerTest extends ApiTestCase { - public function testStoreDeprecated() + public function testGet() { - $id = $this->volume()->id; - $this->doTestApiRoute('POST', "/api/v1/volumes/{$id}/images/metadata"); - } + $volume = $this->volume(); + $id = $volume->id; - public function testStoreImageMetadata() - { - $id = $this->volume()->id; - - $this->doTestApiRoute('POST', "/api/v1/volumes/{$id}/metadata"); + $this->doTestApiRoute('GET', "/api/v1/volumes/{$id}/metadata"); - $csv = new UploadedFile(__DIR__."/../../../../../files/image-metadata.csv", 'image-metadata.csv', 'text/csv', null, true); - $this->beEditor(); - // no permissions - $this->postJson("/api/v1/volumes/{$id}/metadata", ['metadata_csv' => $csv]) + $this->beUser(); + $this->getJson("/api/v1/volumes/{$id}/metadata") ->assertStatus(403); - $this->beAdmin(); - // file required - $this->postJson("/api/v1/volumes/{$id}/metadata")->assertStatus(422); + $this->beGuest(); + $this->getJson("/api/v1/volumes/{$id}/metadata") + ->assertStatus(404); - // image does not exist - $this->postJson("/api/v1/volumes/{$id}/metadata", ['metadata_csv' => $csv]) - ->assertStatus(422); + $disk = Storage::fake('metadata'); + $disk->put($id.'.csv', 'abc'); + $volume->metadata_file_path = $id.'.csv'; + $volume->save(); - $png = ImageTest::create([ - 'filename' => 'abc.png', - 'volume_id' => $id, - ]); - $jpg = ImageTest::create([ - 'filename' => 'abc.jpg', - 'volume_id' => $id, - 'attrs' => ['metadata' => [ - 'water_depth' => 4000, - 'distance_to_ground' => 20, - ]], - ]); - - $this->assertFalse($this->volume()->hasGeoInfo()); - - $this->postJson("/api/v1/volumes/{$id}/metadata", ['metadata_csv' => $csv]) - ->assertStatus(200); + $this->getJson("/api/v1/volumes/-1/metadata") + ->assertStatus(404); - $this->assertTrue($this->volume()->hasGeoInfo()); - - $png = $png->fresh(); - $jpg = $jpg->fresh(); - - $this->assertSame('2016-12-19 12:27:00', $jpg->taken_at->toDateTimeString()); - $this->assertSame(52.220, $jpg->lng); - $this->assertSame(28.123, $jpg->lat); - $this->assertSame(-1500, $jpg->metadata['gps_altitude']); - $this->assertSame(2.6, $jpg->metadata['area']); - // Import should update but not destroy existing metadata. - $this->assertSame(10, $jpg->metadata['distance_to_ground']); - $this->assertSame(4000, $jpg->metadata['water_depth']); - $this->assertSame(180, $jpg->metadata['yaw']); - - $this->assertNull($png->taken_at); - $this->assertNull($png->lng); - $this->assertNull($png->lat); - $this->assertEmpty($png->metadata); + $response = $this->getJson("/api/v1/volumes/{$id}/metadata"); + $response->assertStatus(200); + $this->assertEquals("attachment; filename=biigle-volume-{$id}-metadata.csv", $response->headers->get('content-disposition')); } - public function testStoreStringMetadata() - { - $id = $this->volume()->id; - - $this->beAdmin(); - - $this->postJson("/api/v1/volumes/{$id}/metadata", ['metadata_csv' => "metadata_string"]) - ->assertStatus(422); - } - - public function testStoreDeprecatedFileAttribute() + public function testStoreDeprecated() { $id = $this->volume()->id; - - $image = ImageTest::create([ - 'filename' => 'abc.jpg', - 'volume_id' => $id, - 'attrs' => ['metadata' => [ - 'water_depth' => 4000, - 'distance_to_ground' => 20, - ]], - ]); - - $csv = new UploadedFile(__DIR__."/../../../../../files/image-metadata.csv", 'metadata.csv', 'text/csv', null, true); - - $this->beAdmin(); - $this->postJson("/api/v1/volumes/{$id}/metadata", ['file' => $csv]) - ->assertSuccessful(); - - $image->refresh(); - $this->assertSame(4000, $image->metadata['water_depth']); - $this->assertSame(10, $image->metadata['distance_to_ground']); - $this->assertSame(2.6, $image->metadata['area']); + $this->doTestApiRoute('POST', "/api/v1/volumes/{$id}/images/metadata"); } - public function testStoreImageMetadataText() + public function testStoreImageMetadata() { + Storage::fake('metadata'); $id = $this->volume()->id; - $image = ImageTest::create([ - 'filename' => 'abc.jpg', - 'volume_id' => $id, - 'attrs' => ['metadata' => [ - 'water_depth' => 4000, - 'distance_to_ground' => 20, - ]], - ]); + $this->doTestApiRoute('POST', "/api/v1/volumes/{$id}/metadata"); - $this->beAdmin(); + $csv = new UploadedFile(__DIR__."/../../../../../files/image-metadata.csv", 'image-metadata.csv', 'text/csv', null, true); + $this->beEditor(); + // no permissions $this->postJson("/api/v1/volumes/{$id}/metadata", [ - 'metadata_text' => "filename,area,distance_to_ground\nabc.jpg,2.5,10", - ])->assertSuccessful(); - - $image->refresh(); - $this->assertSame(4000, $image->metadata['water_depth']); - $this->assertSame(10, $image->metadata['distance_to_ground']); - $this->assertSame(2.5, $image->metadata['area']); - } - - public function testStoreVideoMetadataCsv() - { - $id = $this->volume()->id; - $this->volume()->media_type_id = MediaType::videoId(); - $this->volume()->save(); - - $video = VideoTest::create([ - 'filename' => 'abc.mp4', - 'volume_id' => $id, - 'taken_at' => ['2016-12-19 12:27:00', '2016-12-19 12:28:00'], - 'attrs' => ['metadata' => [ - 'distance_to_ground' => [20, 120], - ]], - ]); - - $csv = new UploadedFile(__DIR__."/../../../../../files/video-metadata.csv", 'metadata.csv', 'text/csv', null, true); - - $this->beAdmin(); - $this->postJson("/api/v1/volumes/{$id}/metadata", ['file' => $csv]) - ->assertSuccessful(); - - $video->refresh(); - $this->assertSame([10, 5], $video->metadata['distance_to_ground']); - $this->assertSame([180, 181], $video->metadata['yaw']); - } - - public function testStoreVideoMetadataText() - { - $id = $this->volume()->id; - $this->volume()->media_type_id = MediaType::videoId(); - $this->volume()->save(); - - $video = VideoTest::create([ - 'filename' => 'abc.mp4', - 'volume_id' => $id, - 'taken_at' => ['2022-02-24 16:07:00', '2022-02-24 16:08:00'], - 'attrs' => ['metadata' => [ - 'water_depth' => [4000, 4100], - 'distance_to_ground' => [20, 120], - ]], - ]); - - $text = << $csv, + 'parser' => ImageCsvParser::class, + ]) + ->assertStatus(403); $this->beAdmin(); + // file required $this->postJson("/api/v1/volumes/{$id}/metadata", [ - 'metadata_text' => $text, - ])->assertSuccessful(); - - $video->refresh(); - $this->assertCount(2, $video->taken_at); - $this->assertSame([4000, 4100], $video->metadata['water_depth']); - $this->assertSame([10, 150], $video->metadata['distance_to_ground']); - $this->assertSame([2.5, 3.5], $video->metadata['area']); - } - - public function testStoreVideoMetadataMerge() - { - $id = $this->volume()->id; - $this->volume()->media_type_id = MediaType::videoId(); - $this->volume()->save(); - - $video = VideoTest::create([ - 'filename' => 'abc.mp4', - 'volume_id' => $id, - 'taken_at' => ['2022-02-24 16:07:00', '2022-02-24 16:08:00'], - 'attrs' => ['metadata' => [ - 'water_depth' => [4000, 4100], - 'distance_to_ground' => [20, 120], - ]], - ]); - - $text = << ImageCsvParser::class, + ]) + ->assertStatus(422); - $this->beAdmin(); + // parser required $this->postJson("/api/v1/volumes/{$id}/metadata", [ - 'metadata_text' => $text, - ])->assertSuccessful(); - - $video->refresh(); - $this->assertCount(3, $video->taken_at); - $this->assertSame([4000, 4100, null], $video->metadata['water_depth']); - $this->assertSame([20, 120, 150], $video->metadata['distance_to_ground']); - $this->assertSame([2.5, null, 3.5], $video->metadata['area']); - } - - public function testStoreVideoMetadataFillOneVideoButNotTheOther() - { - $id = $this->volume()->id; - $this->volume()->media_type_id = MediaType::videoId(); - $this->volume()->save(); + 'file' => $csv, + ]) + ->assertStatus(422); - $video1 = VideoTest::create([ - 'filename' => 'abc.mp4', - 'volume_id' => $id, - 'taken_at' => ['2022-02-24 16:07:00', '2022-02-24 16:08:00'], - 'attrs' => ['metadata' => [ - 'distance_to_ground' => [20, 120], - ]], - ]); - - $video2 = VideoTest::create([ - 'filename' => 'def.mp4', - 'volume_id' => $id, - 'taken_at' => ['2022-02-24 16:07:00', '2022-02-24 16:08:00'], - 'attrs' => ['metadata' => []], - ]); - - $text = <<beAdmin(); $this->postJson("/api/v1/volumes/{$id}/metadata", [ - 'metadata_text' => $text, - ])->assertSuccessful(); - - $video1->refresh(); - $video2->refresh(); - $this->assertSame([10, 120, 150], $video1->metadata['distance_to_ground']); - $this->assertArrayNotHasKey('distance_to_ground', $video2->metadata); - } - - public function testStoreVideoMetadataCannotUpdateTimestampedWithBasic() - { - $id = $this->volume()->id; - $this->volume()->media_type_id = MediaType::videoId(); - $this->volume()->save(); + 'file' => $csv, + 'parser' => ImageCsvParser::class, + ]) + ->assertStatus(200); - $video = VideoTest::create([ - 'filename' => 'abc.mp4', - 'volume_id' => $id, - 'taken_at' => ['2022-02-24 16:07:00', '2022-02-24 16:08:00'], - 'attrs' => ['metadata' => [ - 'water_depth' => [4000, 4100], - 'distance_to_ground' => [20, 120], - ]], - ]); + Queue::assertPushed(UpdateVolumeMetadata::class, function ($job) { + $this->assertEquals($this->volume()->id, $job->volume->id); - $this->beAdmin(); - // The video has timestamped metadata. There is no way the new area data without - // timestamp can be merged into the timestamped data. - $this->postJson("/api/v1/volumes/{$id}/metadata", [ - 'metadata_text' => "filename,area\nabc.mp4,2.5", - ])->assertStatus(422); + return true; + }); } - public function testStoreVideoMetadataCannotUpdateBasicWithTimestamped() + public function testStoreImageMetadataInvalid() { $id = $this->volume()->id; - $this->volume()->media_type_id = MediaType::videoId(); - $this->volume()->save(); - - $video = VideoTest::create([ - 'filename' => 'abc.mp4', - 'volume_id' => $id, - 'attrs' => ['metadata' => [ - 'water_depth' => [4000], - 'distance_to_ground' => [20], - ]], - ]); - - $text = <<beAdmin(); - // The video has basic metadata. There is no way the new area data with - // timestamp can be merged into the basic data. $this->postJson("/api/v1/volumes/{$id}/metadata", [ - 'metadata_text' => $text, - ])->assertStatus(422); + 'file' => $csv, + 'parser' => ImageCsvParser::class, + ]) + ->assertStatus(422); } - public function testStoreVideoMetadataZerosSingle() + public function testStoreImageMetadataUnknown() { $id = $this->volume()->id; - $this->volume()->media_type_id = MediaType::videoId(); - $this->volume()->save(); - - $video = VideoTest::create([ - 'filename' => 'abc.mp4', - 'volume_id' => $id, - ]); - - $text = <<beAdmin(); $this->postJson("/api/v1/volumes/{$id}/metadata", [ - 'metadata_text' => $text, - ])->assertSuccessful(); - - $video->refresh(); - $this->assertSame([0], $video->metadata['area']); + 'file' => $csv, + 'parser' => 'unknown', + ]) + ->assertStatus(422); } - public function testStoreVideoMetadataZeros() + public function testStoreVideoMetadataCsv() { + Storage::fake('metadata'); $id = $this->volume()->id; $this->volume()->media_type_id = MediaType::videoId(); $this->volume()->save(); - $video = VideoTest::create([ - 'filename' => 'abc.mp4', - 'volume_id' => $id, - ]); - - $text = <<beAdmin(); $this->postJson("/api/v1/volumes/{$id}/metadata", [ - 'metadata_text' => $text, - ])->assertSuccessful(); + 'file' => $csv, + 'parser' => VideoCsvParser::class, + ]) + ->assertSuccessful(); + + Queue::assertPushed(UpdateVolumeMetadata::class, function ($job) { + $this->assertEquals($this->volume()->id, $job->volume->id); - $video->refresh(); - $this->assertSame([0, 1], $video->metadata['area']); + return true; + }); } - public function testStoreVideoMetadataIncorrectEncoding() + public function testStoreMetadataIncorrectEncoding() { $id = $this->volume()->id; - $this->volume()->media_type_id = MediaType::videoId(); - $this->volume()->save(); - - $video = VideoTest::create([ - 'filename' => 'my-video.mp4', - 'volume_id' => $id, - ]); - $csv = new UploadedFile(__DIR__."/../../../../../files/video-metadata-incorrect-encoding.csv", 'metadata.csv', 'text/csv', null, true); + $csv = new UploadedFile(__DIR__."/../../../../../files/image-metadata-strange-encoding.csv", 'metadata.csv', 'text/csv', null, true); $this->beAdmin(); - $this->postJson("/api/v1/volumes/{$id}/metadata", ['file' => $csv]) + $this->postJson("/api/v1/volumes/{$id}/metadata", [ + 'file' => $csv, + 'parser' => ImageCsvParser::class, + ]) ->assertStatus(422); } - public function testStoreImageIfdoFile() + public function testDestroy() { - $id = $this->volume()->id; - $this->beAdmin(); - $file = new UploadedFile(__DIR__."/../../../../../files/image-ifdo.yaml", 'ifdo.yaml', 'application/yaml', null, true); - - Storage::fake('ifdos'); + $volume = $this->volume(); + $id = $volume->id; - $this->assertFalse($this->volume()->hasIfdo()); + $this->doTestApiRoute('DELETE', "/api/v1/volumes/{$id}/metadata"); - $this->postJson("/api/v1/volumes/{$id}/metadata", ['ifdo_file' => $file]) - ->assertSuccessful(); + $disk = Storage::fake('metadata'); + $disk->put($id.'.csv', 'abc'); + $volume->metadata_file_path = $id.'.csv'; + $volume->save(); - $this->assertTrue($this->volume()->hasIfdo()); - } + $this->beExpert(); + $this->deleteJson("/api/v1/volumes/{$id}/metadata") + ->assertStatus(403); - public function testStoreVideoIfdoFile() - { - $id = $this->volume()->id; - $this->volume()->media_type_id = MediaType::videoId(); - $this->volume()->save(); $this->beAdmin(); - $file = new UploadedFile(__DIR__."/../../../../../files/video-ifdo.yaml", 'ifdo.yaml', 'application/yaml', null, true); - - Storage::fake('ifdos'); - - $this->assertFalse($this->volume()->hasIfdo()); - - $this->postJson("/api/v1/volumes/{$id}/metadata", ['ifdo_file' => $file]) + $this->deleteJson("/api/v1/volumes/{$id}/metadata") ->assertSuccessful(); - $this->assertTrue($this->volume()->hasIfdo()); - } - - public function testStoreVideoIfdoFileForImageVolume() - { - $id = $this->volume()->id; - $this->beAdmin(); - $file = new UploadedFile(__DIR__."/../../../../../files/video-ifdo.yaml", 'ifdo.yaml', 'application/yaml', null, true); - - $this->postJson("/api/v1/volumes/{$id}/metadata", ['ifdo_file' => $file]) - ->assertStatus(422); - } - - public function testStoreImageIfdoFileForVideoVolume() - { - $id = $this->volume()->id; - $this->volume()->media_type_id = MediaType::videoId(); - $this->volume()->save(); - $this->beAdmin(); - $file = new UploadedFile(__DIR__."/../../../../../files/image-ifdo.yaml", 'ifdo.yaml', 'application/yaml', null, true); - - $this->postJson("/api/v1/volumes/{$id}/metadata", ['ifdo_file' => $file]) - ->assertStatus(422); + $volume->refresh(); + $this->assertFalse($volume->hasMetadata()); + $this->assertNull($volume->parser); } } diff --git a/tests/php/Http/Controllers/Api/Volumes/ParseIfdoControllerTest.php b/tests/php/Http/Controllers/Api/Volumes/ParseIfdoControllerTest.php deleted file mode 100644 index cb7edc016..000000000 --- a/tests/php/Http/Controllers/Api/Volumes/ParseIfdoControllerTest.php +++ /dev/null @@ -1,41 +0,0 @@ -doTestApiRoute('POST', "/api/v1/volumes/parse-ifdo"); - - $this->beUser(); - $this->postJson("/api/v1/volumes/parse-ifdo") - ->assertStatus(422); - - $file = new UploadedFile(__DIR__."/../../../../../files/image-metadata.csv", 'metadata.csv', 'text/csv', null, true); - - $this->postJson("/api/v1/volumes/parse-ifdo", ['file' => $file]) - ->assertStatus(422); - - $file = new UploadedFile(__DIR__."/../../../../../files/image-ifdo.yaml", 'ifdo.yaml', 'application/yaml', null, true); - $expect = [ - 'name' => 'SO268 SO268-2_100-1_OFOS SO_CAM-1_Photo_OFOS', - 'url' => 'https://hdl.handle.net/20.500.12085/d7546c4b-307f-4d42-8554-33236c577450@data', - 'handle' => '20.500.12085/d7546c4b-307f-4d42-8554-33236c577450', - 'uuid' => 'd7546c4b-307f-4d42-8554-33236c577450', - 'media_type' => 'image', - 'files' => [ - ['filename', 'area', 'distance_to_ground', 'gps_altitude', 'lat', 'lng', 'taken_at', 'yaw'], - ['SO268-2_100-1_OFOS_SO_CAM-1_20190406_042927.JPG', 5.0, 2, -2248.0, 11.8581802, -117.0214864, '2019-04-06 04:29:27.000000', 20], - ['SO268-2_100-1_OFOS_SO_CAM-1_20190406_052726.JPG', 5.1, 2.1, -4129.6, 11.8582192, -117.0214286, '2019-04-06 05:27:26.000000', 21], - ], - ]; - - $this->postJson("/api/v1/volumes/parse-ifdo", ['file' => $file]) - ->assertStatus(200) - ->assertExactJson($expect); - } -} diff --git a/tests/php/Http/Controllers/Views/Volumes/PendingVolumeControllerTest.php b/tests/php/Http/Controllers/Views/Volumes/PendingVolumeControllerTest.php new file mode 100644 index 000000000..0b5c38744 --- /dev/null +++ b/tests/php/Http/Controllers/Views/Volumes/PendingVolumeControllerTest.php @@ -0,0 +1,540 @@ +create([ + 'user_id' => $this->admin()->id, + 'project_id' => $this->project()->id, + ]); + + // not logged in + $this->get("pending-volumes/{$pv->id}")->assertStatus(302); + + // doesn't belong to pending volume + $this->beExpert(); + $this->get("pending-volumes/{$pv->id}")->assertStatus(403); + + $this->beAdmin(); + $this->get("pending-volumes/{$pv->id}")->assertStatus(200); + } + + public function testShowWithVolumeRedirectToSelectAnnotationLabels() + { + $pv = PendingVolume::factory()->create([ + 'user_id' => $this->admin()->id, + 'project_id' => $this->project()->id, + 'volume_id' => $this->volume()->id, + 'import_annotations' => true, + ]); + + $this->beAdmin(); + $this + ->get("pending-volumes/{$pv->id}") + ->assertRedirectToRoute('pending-volume-annotation-labels', $pv->id); + } + + public function testShowWithVolumeRedirectToSelectFileLabels() + { + $pv = PendingVolume::factory()->create([ + 'user_id' => $this->admin()->id, + 'project_id' => $this->project()->id, + 'volume_id' => $this->volume()->id, + 'import_file_labels' => true, + 'only_annotation_labels' => [123], + ]); + + $this->beAdmin(); + $this + ->get("pending-volumes/{$pv->id}") + ->assertRedirectToRoute('pending-volume-file-labels', $pv->id); + } + + public function testShowWithVolumeRedirectToLabelMap() + { + $pv = PendingVolume::factory()->create([ + 'user_id' => $this->admin()->id, + 'project_id' => $this->project()->id, + 'volume_id' => $this->volume()->id, + 'only_annotation_labels' => [123], + 'only_file_labels' => [123], + ]); + + $this->beAdmin(); + $this + ->get("pending-volumes/{$pv->id}") + ->assertRedirectToRoute('pending-volume-label-map', $pv->id); + } + + public function testShowWithVolumeRedirectToLabelMapOnlyAnnotations() + { + $pv = PendingVolume::factory()->create([ + 'user_id' => $this->admin()->id, + 'project_id' => $this->project()->id, + 'volume_id' => $this->volume()->id, + 'only_annotation_labels' => [123], + 'import_annotations' => true, + ]); + + $this->beAdmin(); + $this + ->get("pending-volumes/{$pv->id}") + ->assertRedirectToRoute('pending-volume-label-map', $pv->id); + } + + public function testShowWithVolumeRedirectToLabelMapOnlyFileLabels() + { + $pv = PendingVolume::factory()->create([ + 'user_id' => $this->admin()->id, + 'project_id' => $this->project()->id, + 'volume_id' => $this->volume()->id, + 'only_file_labels' => [123], + 'import_file_labels' => true, + ]); + + $this->beAdmin(); + $this + ->get("pending-volumes/{$pv->id}") + ->assertRedirectToRoute('pending-volume-label-map', $pv->id); + } + + public function testShowWithVolumeRedirectToUserMap() + { + $pv = PendingVolume::factory()->create([ + 'user_id' => $this->admin()->id, + 'project_id' => $this->project()->id, + 'volume_id' => $this->volume()->id, + 'only_annotation_labels' => [123], + 'only_file_labels' => [123], + 'label_map' => ['123' => 456], + ]); + + $this->beAdmin(); + $this + ->get("pending-volumes/{$pv->id}") + ->assertRedirectToRoute('pending-volume-user-map', $pv->id); + } + + public function testShowWithVolumeRedirectToUserMapOnlyAnnotations() + { + $pv = PendingVolume::factory()->create([ + 'user_id' => $this->admin()->id, + 'project_id' => $this->project()->id, + 'volume_id' => $this->volume()->id, + 'only_annotation_labels' => [123], + 'import_annotations' => true, + 'label_map' => ['123' => 456], + ]); + + $this->beAdmin(); + $this + ->get("pending-volumes/{$pv->id}") + ->assertRedirectToRoute('pending-volume-user-map', $pv->id); + } + + public function testShowWithVolumeRedirectToUserMapOnlyFileLabels() + { + $pv = PendingVolume::factory()->create([ + 'user_id' => $this->admin()->id, + 'project_id' => $this->project()->id, + 'volume_id' => $this->volume()->id, + 'only_file_labels' => [123], + 'import_file_labels' => true, + 'label_map' => ['123' => 456], + ]); + + $this->beAdmin(); + $this + ->get("pending-volumes/{$pv->id}") + ->assertRedirectToRoute('pending-volume-user-map', $pv->id); + } + + public function testShowSelectAnnotationLabels() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('1.jpg'); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $annotation = new ImageAnnotation( + shape: Shape::point(), + points: [10, 10], + labels: [$lau], + ); + $file->addAnnotation($annotation); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'user_id' => $this->admin()->id, + 'project_id' => $this->project()->id, + 'volume_id' => $this->volume()->id, + 'metadata_file_path' => 'mymeta.csv', + ]); + + // not logged in + $this->get("pending-volumes/{$pv->id}/annotation-labels")->assertStatus(302); + + // doesn't belong to pending volume + $this->beExpert(); + $this->get("pending-volumes/{$pv->id}/annotation-labels")->assertStatus(403); + + $this->beAdmin(); + $this->get("pending-volumes/{$pv->id}/annotation-labels")->assertStatus(200); + } + + public function testShowSelectAnnotationLabelsNoVolume() + { + $pv = PendingVolume::factory()->create([ + 'user_id' => $this->admin()->id, + 'project_id' => $this->project()->id, + ]); + + $this->beAdmin(); + $this + ->get("pending-volumes/{$pv->id}/annotation-labels") + ->assertRedirectToRoute('pending-volume', $pv->id); + } + + public function testShowSelectAnnotationLabelsNoMetadata() + { + $pv = PendingVolume::factory()->create([ + 'user_id' => $this->admin()->id, + 'project_id' => $this->project()->id, + 'volume_id' => $this->volume()->id, + ]); + + $this->beAdmin(); + $this->get("pending-volumes/{$pv->id}/annotation-labels")->assertStatus(404); + } + + public function testShowSelectAnnotationLabelsNoAnnotations() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('1.jpg'); + $metadata->addFile($file); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'user_id' => $this->admin()->id, + 'project_id' => $this->project()->id, + 'volume_id' => $this->volume()->id, + 'metadata_file_path' => 'mymeta.csv', + ]); + + $this->beAdmin(); + $this->get("pending-volumes/{$pv->id}/annotation-labels")->assertStatus(404); + } + + public function testShowSelectFileLabels() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('1.jpg'); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $file->addFileLabel($lau); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'user_id' => $this->admin()->id, + 'project_id' => $this->project()->id, + 'volume_id' => $this->volume()->id, + 'metadata_file_path' => 'mymeta.csv', + ]); + + // not logged in + $this->get("pending-volumes/{$pv->id}/file-labels")->assertStatus(302); + + // doesn't belong to pending volume + $this->beExpert(); + $this->get("pending-volumes/{$pv->id}/file-labels")->assertStatus(403); + + $this->beAdmin(); + $this->get("pending-volumes/{$pv->id}/file-labels")->assertStatus(200); + } + + public function testShowSelectFileLabelsNoVolume() + { + $pv = PendingVolume::factory()->create([ + 'user_id' => $this->admin()->id, + 'project_id' => $this->project()->id, + ]); + + $this->beAdmin(); + $this + ->get("pending-volumes/{$pv->id}/file-labels") + ->assertRedirectToRoute('pending-volume', $pv->id); + } + + public function testShowSelectFileLabelsNoMetadata() + { + $pv = PendingVolume::factory()->create([ + 'user_id' => $this->admin()->id, + 'project_id' => $this->project()->id, + 'volume_id' => $this->volume()->id, + ]); + + $this->beAdmin(); + $this->get("pending-volumes/{$pv->id}/file-labels")->assertStatus(404); + } + + public function testShowSelectFileLabelsNoFileLabels() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('1.jpg'); + $metadata->addFile($file); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'user_id' => $this->admin()->id, + 'project_id' => $this->project()->id, + 'volume_id' => $this->volume()->id, + 'metadata_file_path' => 'mymeta.csv', + ]); + + $this->beAdmin(); + $this->get("pending-volumes/{$pv->id}/file-labels")->assertStatus(404); + } + + public function testShowSelectLabelMap() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('1.jpg'); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $file->addFileLabel($lau); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'user_id' => $this->admin()->id, + 'project_id' => $this->project()->id, + 'volume_id' => $this->volume()->id, + 'metadata_file_path' => 'mymeta.csv', + ]); + + // not logged in + $this->get("pending-volumes/{$pv->id}/label-map")->assertStatus(302); + + // doesn't belong to pending volume + $this->beExpert(); + $this->get("pending-volumes/{$pv->id}/label-map")->assertStatus(403); + + $this->beAdmin(); + $this->get("pending-volumes/{$pv->id}/label-map")->assertStatus(200); + } + + public function testShowSelectLabelMapNoVolume() + { + $pv = PendingVolume::factory()->create([ + 'user_id' => $this->admin()->id, + 'project_id' => $this->project()->id, + ]); + + $this->beAdmin(); + $this + ->get("pending-volumes/{$pv->id}/label-map") + ->assertRedirectToRoute('pending-volume', $pv->id); + } + + public function testShowSelectLabelMapNoMetadata() + { + $pv = PendingVolume::factory()->create([ + 'user_id' => $this->admin()->id, + 'project_id' => $this->project()->id, + 'volume_id' => $this->volume()->id, + ]); + + $this->beAdmin(); + $this->get("pending-volumes/{$pv->id}/label-map")->assertStatus(404); + } + + public function testShowSelectLabelMapNoLabels() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('1.jpg'); + $metadata->addFile($file); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'user_id' => $this->admin()->id, + 'project_id' => $this->project()->id, + 'volume_id' => $this->volume()->id, + 'metadata_file_path' => 'mymeta.csv', + ]); + + $this->beAdmin(); + $this->get("pending-volumes/{$pv->id}/label-map")->assertStatus(404); + } + + public function testShowSelectUserMap() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('1.jpg'); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $file->addFileLabel($lau); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'user_id' => $this->admin()->id, + 'project_id' => $this->project()->id, + 'volume_id' => $this->volume()->id, + 'metadata_file_path' => 'mymeta.csv', + ]); + + // not logged in + $this->get("pending-volumes/{$pv->id}/user-map")->assertStatus(302); + + // doesn't belong to pending volume + $this->beExpert(); + $this->get("pending-volumes/{$pv->id}/user-map")->assertStatus(403); + + $this->beAdmin(); + $this->get("pending-volumes/{$pv->id}/user-map")->assertStatus(200); + } + + public function testShowSelectUserMapNoVolume() + { + $pv = PendingVolume::factory()->create([ + 'user_id' => $this->admin()->id, + 'project_id' => $this->project()->id, + ]); + + $this->beAdmin(); + $this + ->get("pending-volumes/{$pv->id}/user-map") + ->assertRedirectToRoute('pending-volume', $pv->id); + } + + public function testShowSelectUserMapNoMetadata() + { + $pv = PendingVolume::factory()->create([ + 'user_id' => $this->admin()->id, + 'project_id' => $this->project()->id, + 'volume_id' => $this->volume()->id, + ]); + + $this->beAdmin(); + $this->get("pending-volumes/{$pv->id}/user-map")->assertStatus(404); + } + + public function testShowSelectUserMapNoUsers() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('1.jpg'); + $metadata->addFile($file); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'user_id' => $this->admin()->id, + 'project_id' => $this->project()->id, + 'volume_id' => $this->volume()->id, + 'metadata_file_path' => 'mymeta.csv', + ]); + + $this->beAdmin(); + $this->get("pending-volumes/{$pv->id}/user-map")->assertStatus(404); + } + + public function testShowFinishImport() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('1.jpg'); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $file->addFileLabel($lau); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'user_id' => $this->admin()->id, + 'project_id' => $this->project()->id, + 'volume_id' => $this->volume()->id, + 'metadata_file_path' => 'mymeta.csv', + ]); + + // not logged in + $this->get("pending-volumes/{$pv->id}/finish")->assertStatus(302); + + // doesn't belong to pending volume + $this->beExpert(); + $this->get("pending-volumes/{$pv->id}/finish")->assertStatus(403); + + $this->beAdmin(); + $this->get("pending-volumes/{$pv->id}/finish")->assertStatus(200); + } + + public function testShowFinishImportNoVolume() + { + $pv = PendingVolume::factory()->create([ + 'user_id' => $this->admin()->id, + 'project_id' => $this->project()->id, + ]); + + $this->beAdmin(); + $this + ->get("pending-volumes/{$pv->id}/finish") + ->assertRedirectToRoute('pending-volume', $pv->id); + } + + public function testShowFinishImportNoMetadata() + { + $pv = PendingVolume::factory()->create([ + 'user_id' => $this->admin()->id, + 'project_id' => $this->project()->id, + 'volume_id' => $this->volume()->id, + ]); + + $this->beAdmin(); + $this->get("pending-volumes/{$pv->id}/finish")->assertStatus(404); + } + + public function testShowFinishImportNoUsers() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('1.jpg'); + $metadata->addFile($file); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'user_id' => $this->admin()->id, + 'project_id' => $this->project()->id, + 'volume_id' => $this->volume()->id, + 'metadata_file_path' => 'mymeta.csv', + ]); + + $this->beAdmin(); + $this->get("pending-volumes/{$pv->id}/finish")->assertStatus(404); + } +} diff --git a/tests/php/Http/Controllers/Views/Volumes/VolumeControllerTest.php b/tests/php/Http/Controllers/Views/Volumes/VolumeControllerTest.php index 91638805e..8fd78f228 100644 --- a/tests/php/Http/Controllers/Views/Volumes/VolumeControllerTest.php +++ b/tests/php/Http/Controllers/Views/Volumes/VolumeControllerTest.php @@ -3,6 +3,7 @@ namespace Biigle\Tests\Http\Controllers\Views\Volumes; use ApiTestCase; +use Biigle\PendingVolume; class VolumeControllerTest extends ApiTestCase { @@ -62,6 +63,21 @@ public function testCreate() $response->assertStatus(200); } + public function testCreateWithExistingRedirectToStep2() + { + $pv = PendingVolume::factory()->create([ + 'project_id' => $this->project()->id, + 'user_id' => $this->admin()->id, + ]); + + $id = $this->project()->id; + $this->beAdmin(); + + $this + ->get('volumes/create?project='.$id) + ->assertRedirectToRoute('pending-volume', $pv->id); + } + public function testEdit() { $id = $this->volume()->id; diff --git a/tests/php/Jobs/CloneImagesOrVideosTest.php b/tests/php/Jobs/CloneImagesOrVideosTest.php index 9d1f528ec..73bf8eb23 100644 --- a/tests/php/Jobs/CloneImagesOrVideosTest.php +++ b/tests/php/Jobs/CloneImagesOrVideosTest.php @@ -2,9 +2,11 @@ namespace Biigle\Tests\Jobs; +use ApiTestCase; use Biigle\Jobs\CloneImagesOrVideos; use Biigle\Jobs\ProcessNewVolumeFiles; use Biigle\MediaType; +use Biigle\Services\MetadataParsing\ImageCsvParser; use Biigle\Tests\ImageAnnotationLabelTest; use Biigle\Tests\ImageAnnotationTest; use Biigle\Tests\ImageLabelTest; @@ -22,7 +24,7 @@ use Illuminate\Support\Facades\Queue; use Illuminate\Support\Facades\Storage; -class CloneImagesOrVideosTest extends \ApiTestCase +class CloneImagesOrVideosTest extends ApiTestCase { public function testCloneImageVolume() { @@ -570,23 +572,24 @@ public function testCloneVolumeVideoWithoutAnnotations() $this->assertEmpty($newVideo->annotations()->get()); } - public function testCloneVolumeIfDoFiles() + public function testCloneVolumeMetadataFile() { - Event::fake(); + Storage::fake('metadata'); $volume = $this->volume([ 'media_type_id' => MediaType::imageId(), 'created_at' => '2022-11-09 14:37:00', 'updated_at' => '2022-11-09 14:37:00', + 'metadata_parser' => ImageCsvParser::class, ])->fresh(); $copy = $volume->replicate(); + $copy->metadata_file_path = 'mymeta.csv'; $copy->save(); // Use fresh() to load even the null fields. - Storage::fake('ifdos'); - $csv = __DIR__."/../../files/image-ifdo.yaml"; - $file = new UploadedFile($csv, 'ifdo.yaml', 'application/yaml', null, true); - $volume->saveIfdo($file); + $csv = __DIR__."/../../files/image-metadata.csv"; + $file = new UploadedFile($csv, 'metadata.csv', 'text/csv', null, true); + $volume->saveMetadata($file); // The target project. $project = ProjectTest::create(); @@ -595,12 +598,11 @@ public function testCloneVolumeIfDoFiles() $request = new Request(['project' => $project, 'volume' => $volume]); with(new CloneImagesOrVideos($request, $copy))->handle(); - Event::assertDispatched('volume.cloned'); $copy = $project->volumes()->first(); - $this->assertNotNull($copy->getIfdo()); - $this->assertTrue($copy->hasIfdo()); - $this->assertSame($volume->getIfdo(), $copy->getIfdo()); + $this->assertTrue($copy->hasMetadata()); + $this->assertNotNull($copy->getMetadata()); + $this->assertSame(ImageCsvParser::class, $copy->metadata_parser); } public function testHandleVolumeImages() diff --git a/tests/php/Jobs/CreateNewImagesOrVideosTest.php b/tests/php/Jobs/CreateNewImagesOrVideosTest.php index fc7ee3abd..c53219f8d 100644 --- a/tests/php/Jobs/CreateNewImagesOrVideosTest.php +++ b/tests/php/Jobs/CreateNewImagesOrVideosTest.php @@ -5,10 +5,13 @@ use Biigle\Jobs\CreateNewImagesOrVideos; use Biigle\Jobs\ProcessNewVolumeFiles; use Biigle\MediaType; +use Biigle\Services\MetadataParsing\ImageCsvParser; +use Biigle\Services\MetadataParsing\VideoCsvParser; use Biigle\Tests\VolumeTest; use Carbon\Carbon; use Illuminate\Support\Facades\Event; use Illuminate\Support\Facades\Queue; +use Illuminate\Support\Facades\Storage; use TestCase; class CreateNewImagesOrVideosTest extends TestCase @@ -65,14 +68,17 @@ public function testHandleImageMetadata() { $volume = VolumeTest::create([ 'media_type_id' => MediaType::imageId(), + 'metadata_file_path' => 'mymeta.csv', + 'metadata_parser' => ImageCsvParser::class, ]); + $disk = Storage::fake('metadata'); + $disk->put($volume->metadata_file_path, <<handle(); + with(new CreateNewImagesOrVideos($volume, $filenames))->handle(); $image = $volume->images()->first(); $this->assertSame('2016-12-19 12:27:00', $image->taken_at->toDateTimeString()); $this->assertSame(52.220, $image->lng); @@ -87,14 +93,17 @@ public function testHandleImageMetadataEmptyCells() { $volume = VolumeTest::create([ 'media_type_id' => MediaType::imageId(), + 'metadata_file_path' => 'mymeta.csv', + 'metadata_parser' => ImageCsvParser::class, ]); + $disk = Storage::fake('metadata'); + $disk->put($volume->metadata_file_path, <<handle(); + with(new CreateNewImagesOrVideos($volume, $filenames))->handle(); $image = $volume->images()->first(); $this->assertSame(52.220, $image->lng); $this->assertSame(28.123, $image->lat); @@ -105,14 +114,17 @@ public function testHandleImageMetadataIncomplete() { $volume = VolumeTest::create([ 'media_type_id' => MediaType::imageId(), + 'metadata_file_path' => 'mymeta.csv', + 'metadata_parser' => ImageCsvParser::class, ]); + $disk = Storage::fake('metadata'); + $disk->put($volume->metadata_file_path, <<handle(); + with(new CreateNewImagesOrVideos($volume, $filenames))->handle(); $this->assertSame(2, $volume->images()->count()); } @@ -120,16 +132,18 @@ public function testHandleVideoMetadata() { $volume = VolumeTest::create([ 'media_type_id' => MediaType::videoId(), + 'metadata_file_path' => 'mymeta.csv', + 'metadata_parser' => VideoCsvParser::class, ]); + $disk = Storage::fake('metadata'); + $disk->put($volume->metadata_file_path, <<handle(); + with(new CreateNewImagesOrVideos($volume, $filenames))->handle(); $video = $volume->videos()->first(); $expect = [ Carbon::parse('2016-12-19 12:27:00'), @@ -148,15 +162,18 @@ public function testHandleVideoMetadataEmptyCells() { $volume = VolumeTest::create([ 'media_type_id' => MediaType::videoId(), + 'metadata_file_path' => 'mymeta.csv', + 'metadata_parser' => VideoCsvParser::class, ]); + $disk = Storage::fake('metadata'); + $disk->put($volume->metadata_file_path, <<handle(); + with(new CreateNewImagesOrVideos($volume, $filenames))->handle(); $video = $volume->videos()->first(); $expect = ['gps_altitude' => [-1500, null]]; $this->assertSame($expect, $video->metadata); @@ -166,14 +183,17 @@ public function testHandleVideoMetadataZeroSingle() { $volume = VolumeTest::create([ 'media_type_id' => MediaType::videoId(), + 'metadata_file_path' => 'mymeta.csv', + 'metadata_parser' => VideoCsvParser::class, ]); + $disk = Storage::fake('metadata'); + $disk->put($volume->metadata_file_path, <<handle(); + with(new CreateNewImagesOrVideos($volume, $filenames))->handle(); $video = $volume->videos()->first(); $expect = ['distance_to_ground' => [0]]; $this->assertSame($expect, $video->metadata); @@ -183,15 +203,18 @@ public function testHandleVideoMetadataZero() { $volume = VolumeTest::create([ 'media_type_id' => MediaType::videoId(), + 'metadata_file_path' => 'mymeta.csv', + 'metadata_parser' => VideoCsvParser::class, ]); + $disk = Storage::fake('metadata'); + $disk->put($volume->metadata_file_path, <<handle(); + with(new CreateNewImagesOrVideos($volume, $filenames))->handle(); $video = $volume->videos()->first(); $expect = ['distance_to_ground' => [0, 1]]; $this->assertSame($expect, $video->metadata); @@ -201,14 +224,17 @@ public function testHandleVideoMetadataBasic() { $volume = VolumeTest::create([ 'media_type_id' => MediaType::videoId(), + 'metadata_file_path' => 'mymeta.csv', + 'metadata_parser' => VideoCsvParser::class, ]); + $disk = Storage::fake('metadata'); + $disk->put($volume->metadata_file_path, <<handle(); + with(new CreateNewImagesOrVideos($volume, $filenames))->handle(); $video = $volume->videos()->first(); $expect = ['gps_altitude' => [-1500]]; $this->assertSame($expect, $video->metadata); @@ -219,14 +245,17 @@ public function testHandleVideoMetadataIncomplete() { $volume = VolumeTest::create([ 'media_type_id' => MediaType::videoId(), + 'metadata_file_path' => 'mymeta.csv', + 'metadata_parser' => VideoCsvParser::class, ]); + $disk = Storage::fake('metadata'); + $disk->put($volume->metadata_file_path, <<handle(); + with(new CreateNewImagesOrVideos($volume, $filenames))->handle(); $this->assertSame(2, $volume->videos()->count()); } @@ -234,14 +263,17 @@ public function testHandleMetadataDateParsing() { $volume = VolumeTest::create([ 'media_type_id' => MediaType::imageId(), + 'metadata_file_path' => 'mymeta.csv', + 'metadata_parser' => VideoCsvParser::class, ]); + $disk = Storage::fake('metadata'); + $disk->put($volume->metadata_file_path, <<handle(); + with(new CreateNewImagesOrVideos($volume, $filenames))->handle(); $image = $volume->images()->first(); $this->assertSame('2019-05-01 10:35:00', $image->taken_at->toDateTimeString()); } diff --git a/tests/php/Jobs/ImportVolumeMetadataTest.php b/tests/php/Jobs/ImportVolumeMetadataTest.php new file mode 100644 index 000000000..5fbbfaf73 --- /dev/null +++ b/tests/php/Jobs/ImportVolumeMetadataTest.php @@ -0,0 +1,535 @@ +create(); + $dbUser = DbUser::factory()->create(); + $dbLabel = DbLabel::factory()->create(); + + $metadata = new VolumeMetadata; + $file = new ImageMetadata($image->filename); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $file->addFileLabel($lau); + $annotation = new ImageAnnotation( + shape: Shape::point(), + points: [10, 10], + labels: [$lau], + ); + $file->addAnnotation($annotation); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'media_type_id' => MediaType::imageId(), + 'metadata_file_path' => 'mymeta.csv', + 'import_file_labels' => true, + 'import_annotations' => true, + 'user_map' => [321 => $dbUser->id], + 'label_map' => [123 => $dbLabel->id], + 'volume_id' => $image->volume_id, + ]); + + (new ImportVolumeMetadata($pv))->handle(); + + $annotations = $image->annotations; + $this->assertCount(1, $annotations); + $this->assertEquals([10, 10], $annotations[0]->points); + $this->assertEquals(Shape::pointId(), $annotations[0]->shape_id); + $this->assertNotNull($annotations[0]->created_at); + $this->assertNotNull($annotations[0]->updated_at); + $annotationLabels = $annotations[0]->labels; + $this->assertCount(1, $annotationLabels); + $this->assertEquals($dbUser->id, $annotationLabels[0]->user_id); + $this->assertEquals($dbLabel->id, $annotationLabels[0]->label_id); + $this->assertNotNull($annotationLabels[0]->created_at); + $this->assertNotNull($annotationLabels[0]->updated_at); + + $fileLabels = $image->labels; + $this->assertCount(1, $fileLabels); + $this->assertEquals($dbUser->id, $fileLabels[0]->user_id); + $this->assertEquals($dbLabel->id, $fileLabels[0]->label_id); + + $this->assertNull($pv->fresh()); + } + + public function testHandleVideo() + { + $video = Video::factory()->create(); + $dbUser = DbUser::factory()->create(); + $dbLabel = DbLabel::factory()->create(); + + $metadata = new VolumeMetadata; + $file = new ImageMetadata($video->filename); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $file->addFileLabel($lau); + $annotation = new VideoAnnotation( + shape: Shape::point(), + points: [[10, 10]], + frames: [1], + labels: [$lau], + ); + $file->addAnnotation($annotation); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'media_type_id' => MediaType::videoId(), + 'metadata_file_path' => 'mymeta.csv', + 'import_file_labels' => true, + 'import_annotations' => true, + 'user_map' => [321 => $dbUser->id], + 'label_map' => [123 => $dbLabel->id], + 'volume_id' => $video->volume_id, + ]); + + (new ImportVolumeMetadata($pv))->handle(); + + $annotations = $video->annotations; + $this->assertCount(1, $annotations); + $this->assertEquals([[10, 10]], $annotations[0]->points); + $this->assertEquals([1], $annotations[0]->frames); + $this->assertEquals(Shape::pointId(), $annotations[0]->shape_id); + $this->assertNotNull($annotations[0]->created_at); + $this->assertNotNull($annotations[0]->updated_at); + $annotationLabels = $annotations[0]->labels; + $this->assertCount(1, $annotationLabels); + $this->assertEquals($dbUser->id, $annotationLabels[0]->user_id); + $this->assertEquals($dbLabel->id, $annotationLabels[0]->label_id); + $this->assertNotNull($annotationLabels[0]->created_at); + $this->assertNotNull($annotationLabels[0]->updated_at); + + $fileLabels = $video->labels; + $this->assertCount(1, $fileLabels); + $this->assertEquals($dbUser->id, $fileLabels[0]->user_id); + $this->assertEquals($dbLabel->id, $fileLabels[0]->label_id); + + $this->assertNull($pv->fresh()); + } + + public function testHandleMatchByUuid() + { + $image = Image::factory()->create(); + $dbUser = DbUser::factory()->create(); + $dbLabel = DbLabel::factory()->create(); + + $metadata = new VolumeMetadata; + $file = new ImageMetadata($image->filename); + $metadata->addFile($file); + $label = new Label(123, 'my label', uuid: $dbLabel->uuid); + $user = new User(321, 'joe user', uuid: $dbUser->uuid); + $lau = new LabelAndUser($label, $user); + $file->addFileLabel($lau); + $annotation = new ImageAnnotation( + shape: Shape::point(), + points: [10, 10], + labels: [$lau], + ); + $file->addAnnotation($annotation); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'media_type_id' => MediaType::imageId(), + 'metadata_file_path' => 'mymeta.csv', + 'import_file_labels' => true, + 'import_annotations' => true, + 'volume_id' => $image->volume_id, + ]); + + (new ImportVolumeMetadata($pv))->handle(); + + $annotation = $image->annotations()->first(); + $this->assertEquals($dbUser->id, $annotation->labels[0]->user_id); + $this->assertEquals($dbLabel->id, $annotation->labels[0]->label_id); + + $this->assertEquals($dbUser->id, $image->labels[0]->user_id); + $this->assertEquals($dbLabel->id, $image->labels[0]->label_id); + } + + public function testHandleDeletedUser() + { + $image = Image::factory()->create(); + $dbUser = DbUser::factory()->create(); + $dbLabel = DbLabel::factory()->create(); + + $metadata = new VolumeMetadata; + $file = new ImageMetadata($image->filename); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $file->addFileLabel($lau); + $annotation = new ImageAnnotation( + shape: Shape::point(), + points: [10, 10], + labels: [$lau], + ); + $file->addAnnotation($annotation); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'media_type_id' => MediaType::imageId(), + 'metadata_file_path' => 'mymeta.csv', + 'import_file_labels' => true, + 'import_annotations' => true, + 'user_map' => [321 => -1], + 'label_map' => [123 => $dbLabel->id], + 'volume_id' => $image->volume_id, + ]); + + (new ImportVolumeMetadata($pv))->handle(); + + $annotation = $image->annotations()->first(); + $this->assertNull($annotation->labels[0]->user_id); + $this->assertNull($image->labels[0]->user_id); + } + + public function testHandleDeletedLabel() + { + $image = Image::factory()->create(); + $dbUser = DbUser::factory()->create(); + $dbLabel = DbLabel::factory()->create(); + + $metadata = new VolumeMetadata; + $file = new ImageMetadata($image->filename); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $file->addFileLabel($lau); + $annotation = new ImageAnnotation( + shape: Shape::point(), + points: [10, 10], + labels: [$lau], + ); + $file->addAnnotation($annotation); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'media_type_id' => MediaType::imageId(), + 'metadata_file_path' => 'mymeta.csv', + 'import_file_labels' => true, + 'import_annotations' => true, + 'user_map' => [321 => $dbUser->id], + 'label_map' => [123 => -1], + 'volume_id' => $image->volume_id, + ]); + + (new ImportVolumeMetadata($pv))->handle(); + + $this->assertFalse($image->annotations()->exists()); + $this->assertFalse($image->labels()->exists()); + } + + public function testHandleIgnoreAnnotations() + { + $image = Image::factory()->create(); + $dbUser = DbUser::factory()->create(); + $dbLabel = DbLabel::factory()->create(); + + $metadata = new VolumeMetadata; + $file = new ImageMetadata($image->filename); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $file->addFileLabel($lau); + $annotation = new ImageAnnotation( + shape: Shape::point(), + points: [10, 10], + labels: [$lau], + ); + $file->addAnnotation($annotation); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'media_type_id' => MediaType::imageId(), + 'metadata_file_path' => 'mymeta.csv', + 'import_file_labels' => true, + 'import_annotations' => false, + 'user_map' => [321 => $dbUser->id], + 'label_map' => [123 => $dbLabel->id], + 'volume_id' => $image->volume_id, + ]); + + (new ImportVolumeMetadata($pv))->handle(); + + $this->assertFalse($image->annotations()->exists()); + $this->assertTrue($image->labels()->exists()); + } + + public function testHandleIgnoreFileLabels() + { + $image = Image::factory()->create(); + $dbUser = DbUser::factory()->create(); + $dbLabel = DbLabel::factory()->create(); + + $metadata = new VolumeMetadata; + $file = new ImageMetadata($image->filename); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $file->addFileLabel($lau); + $annotation = new ImageAnnotation( + shape: Shape::point(), + points: [10, 10], + labels: [$lau], + ); + $file->addAnnotation($annotation); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'media_type_id' => MediaType::imageId(), + 'metadata_file_path' => 'mymeta.csv', + 'import_file_labels' => false, + 'import_annotations' => true, + 'user_map' => [321 => $dbUser->id], + 'label_map' => [123 => $dbLabel->id], + 'volume_id' => $image->volume_id, + ]); + + (new ImportVolumeMetadata($pv))->handle(); + + $this->assertTrue($image->annotations()->exists()); + $this->assertFalse($image->labels()->exists()); + } + + public function testHandleFilterAnnotationLabels() + { + $image = Image::factory()->create(); + $dbUser = DbUser::factory()->create(); + $dbLabel1 = DbLabel::factory()->create(); + $dbLabel2 = DbLabel::factory()->create(); + + $metadata = new VolumeMetadata; + $file = new ImageMetadata($image->filename); + $metadata->addFile($file); + $label1 = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label1, $user); + $file->addFileLabel($lau); + $annotation = new ImageAnnotation( + shape: Shape::point(), + points: [10, 10], + labels: [$lau], + ); + $file->addAnnotation($annotation); + + $label2 = new Label(234, 'my label'); + $lau = new LabelAndUser($label2, $user); + $file->addFileLabel($lau); + $annotation = new ImageAnnotation( + shape: Shape::point(), + points: [20, 20], + labels: [$lau], + ); + $file->addAnnotation($annotation); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'media_type_id' => MediaType::imageId(), + 'metadata_file_path' => 'mymeta.csv', + 'import_file_labels' => true, + 'import_annotations' => true, + 'user_map' => [321 => $dbUser->id], + 'label_map' => [123 => $dbLabel1->id, 234 => $dbLabel2->id], + 'only_annotation_labels' => [123], + 'volume_id' => $image->volume_id, + ]); + + (new ImportVolumeMetadata($pv))->handle(); + + $annotations = $image->annotations; + $this->assertCount(1, $annotations); + $this->assertEquals([10, 10], $annotations[0]->points); + $this->assertEquals(Shape::pointId(), $annotations[0]->shape_id); + $annotationLabels = $annotations[0]->labels; + $this->assertCount(1, $annotationLabels); + $this->assertEquals($dbLabel1->id, $annotationLabels[0]->label_id); + + $this->assertEquals(2, $image->labels()->count()); + } + + public function testHandleFilterFileLabels() + { + $image = Image::factory()->create(); + $dbUser = DbUser::factory()->create(); + $dbLabel1 = DbLabel::factory()->create(); + $dbLabel2 = DbLabel::factory()->create(); + + $metadata = new VolumeMetadata; + $file = new ImageMetadata($image->filename); + $metadata->addFile($file); + $label1 = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label1, $user); + $file->addFileLabel($lau); + $annotation = new ImageAnnotation( + shape: Shape::point(), + points: [10, 10], + labels: [$lau], + ); + $file->addAnnotation($annotation); + + $label2 = new Label(234, 'my label'); + $lau = new LabelAndUser($label2, $user); + $file->addFileLabel($lau); + $annotation = new ImageAnnotation( + shape: Shape::point(), + points: [20, 20], + labels: [$lau], + ); + $file->addAnnotation($annotation); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'media_type_id' => MediaType::imageId(), + 'metadata_file_path' => 'mymeta.csv', + 'import_file_labels' => true, + 'import_annotations' => true, + 'user_map' => [321 => $dbUser->id], + 'label_map' => [123 => $dbLabel1->id, 234 => $dbLabel2->id], + 'only_file_labels' => [123], + 'volume_id' => $image->volume_id, + ]); + + (new ImportVolumeMetadata($pv))->handle(); + + $this->assertEquals(2, $image->annotations()->count()); + + $fileLabels = $image->labels; + $this->assertCount(1, $fileLabels); + $this->assertEquals($dbLabel1->id, $fileLabels[0]->label_id); + } + + public function testHandleIgnoreMissingFiles() + { + $image = Image::factory()->create(); + $dbUser = DbUser::factory()->create(); + $dbLabel = DbLabel::factory()->create(); + + $metadata = new VolumeMetadata; + $file = new ImageMetadata('unknown.jpg'); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $file->addFileLabel($lau); + $annotation = new ImageAnnotation( + shape: Shape::point(), + points: [10, 10], + labels: [$lau], + ); + $file->addAnnotation($annotation); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'media_type_id' => MediaType::imageId(), + 'metadata_file_path' => 'mymeta.csv', + 'import_file_labels' => true, + 'import_annotations' => true, + 'user_map' => [321 => $dbUser->id], + 'label_map' => [123 => $dbLabel->id], + 'volume_id' => $image->volume_id, + ]); + + (new ImportVolumeMetadata($pv))->handle(); + + $this->assertFalse($image->annotations()->exists()); + $this->assertFalse($image->labels()->exists()); + } + + public function testHandleRetryWhileCreatingAsync() + { + $image = Image::factory()->create(); + $volume = Volume::factory()->create(); + $volume->creating_async = true; + $volume->save(); + + $pv = PendingVolume::factory()->create(['volume_id' => $volume->id]); + + (new ImportVolumeMetadata($pv))->handle(); + + $this->assertNotNull($pv->fresh()); + } + + public function testHandleChunkAnnotations() + { + ImportVolumeMetadata::$insertChunkSize = 1; + + $image = Image::factory()->create(); + $dbUser = DbUser::factory()->create(); + $dbLabel = DbLabel::factory()->create(); + + $metadata = new VolumeMetadata; + $file = new ImageMetadata($image->filename); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $annotation = new ImageAnnotation( + shape: Shape::point(), + points: [10, 10], + labels: [$lau], + ); + $file->addAnnotation($annotation); + $annotation = new ImageAnnotation( + shape: Shape::point(), + points: [20, 20], + labels: [$lau], + ); + $file->addAnnotation($annotation); + + Cache::store('array')->put('metadata-pending-metadata-mymeta.csv', $metadata); + + $pv = PendingVolume::factory()->create([ + 'media_type_id' => MediaType::imageId(), + 'metadata_file_path' => 'mymeta.csv', + 'import_annotations' => true, + 'user_map' => [321 => $dbUser->id], + 'label_map' => [123 => $dbLabel->id], + 'volume_id' => $image->volume_id, + ]); + + (new ImportVolumeMetadata($pv))->handle(); + + $this->assertEquals(2, $image->annotations()->count()); + } +} diff --git a/tests/php/Jobs/UpdateVolumeMetadataTest.php b/tests/php/Jobs/UpdateVolumeMetadataTest.php new file mode 100644 index 000000000..d010edc2c --- /dev/null +++ b/tests/php/Jobs/UpdateVolumeMetadataTest.php @@ -0,0 +1,301 @@ +create([ + 'media_type_id' => MediaType::imageId(), + 'metadata_file_path' => 'mymeta.csv', + 'metadata_parser' => ImageCsvParser::class, + ]); + + $image = Image::factory()->create([ + 'filename' => 'a.jpg', + 'volume_id' => $volume->id, + 'attrs' => [ + 'size' => 100, + ], + ]); + + + $disk = Storage::fake('metadata'); + $disk->put($volume->metadata_file_path, <<assertFalse($volume->hasGeoInfo()); + + with(new UpdateVolumeMetadata($volume))->handle(); + $image->refresh(); + $this->assertEquals(100, $image->size); + $this->assertEquals('2016-12-19 12:27:00', $image->taken_at); + $this->assertEquals(52.220, $image->lng); + $this->assertEquals(28.123, $image->lat); + $this->assertEquals(-1500, $image->metadata['gps_altitude']); + $this->assertEquals(10, $image->metadata['distance_to_ground']); + $this->assertEquals(2.6, $image->metadata['area']); + $this->assertEquals(180, $image->metadata['yaw']); + $this->assertTrue($volume->hasGeoInfo()); + } + + public function testHandleImageUpdate() + { + $volume = Volume::factory()->create([ + 'media_type_id' => MediaType::imageId(), + 'metadata_file_path' => 'mymeta.csv', + 'metadata_parser' => ImageCsvParser::class, + ]); + + $image = Image::factory()->create([ + 'filename' => 'a.jpg', + 'volume_id' => $volume->id, + 'taken_at' => '2024-03-12 11:23:00', + 'lng' => 12, + 'lat' => 34, + 'attrs' => [ + 'size' => 100, + 'metadata' => [ + 'gps_altitude' => -1000, + 'distance_to_ground' => 5, + 'area' => 2.5, + 'yaw' => 100, + ], + ], + ]); + + + $disk = Storage::fake('metadata'); + $disk->put($volume->metadata_file_path, <<handle(); + $image->refresh(); + $this->assertEquals(100, $image->size); + $this->assertEquals('2016-12-19 12:27:00', $image->taken_at); + $this->assertEquals(52.220, $image->lng); + $this->assertEquals(28.123, $image->lat); + $this->assertEquals(-1500, $image->metadata['gps_altitude']); + $this->assertEquals(10, $image->metadata['distance_to_ground']); + $this->assertEquals(2.6, $image->metadata['area']); + $this->assertEquals(180, $image->metadata['yaw']); + } + + public function testHandleImageMerge() + { + $volume = Volume::factory()->create([ + 'media_type_id' => MediaType::imageId(), + 'metadata_file_path' => 'mymeta.csv', + 'metadata_parser' => ImageCsvParser::class, + ]); + + $image = Image::factory()->create([ + 'filename' => 'a.jpg', + 'volume_id' => $volume->id, + 'lng' => 12, + 'lat' => 34, + 'attrs' => [ + 'size' => 100, + 'metadata' => [ + 'gps_altitude' => -1000, + 'distance_to_ground' => 5, + ], + ], + ]); + + + $disk = Storage::fake('metadata'); + $disk->put($volume->metadata_file_path, <<handle(); + $image->refresh(); + $this->assertEquals(100, $image->size); + $this->assertEquals('2016-12-19 12:27:00', $image->taken_at); + $this->assertEquals(12, $image->lng); + $this->assertEquals(34, $image->lat); + $this->assertEquals(-1500, $image->metadata['gps_altitude']); + $this->assertEquals(5, $image->metadata['distance_to_ground']); + $this->assertArrayNotHasKey('area', $image->metadata); + $this->assertArrayNotHasKey('yaw', $image->metadata); + } + + public function testHandleVideoAdd() + { + $volume = Volume::factory()->create([ + 'media_type_id' => MediaType::videoId(), + 'metadata_file_path' => 'mymeta.csv', + 'metadata_parser' => VideoCsvParser::class, + ]); + + $video = Video::factory()->create([ + 'filename' => 'a.mp4', + 'volume_id' => $volume->id, + 'attrs' => [ + 'size' => 100, + ], + ]); + + + $disk = Storage::fake('metadata'); + $disk->put($volume->metadata_file_path, <<handle(); + $video->refresh(); + $this->assertEquals(100, $video->size); + $this->assertEquals([Carbon::parse('2016-12-19 12:27:00')], $video->taken_at); + $this->assertEquals([52.220], $video->lng); + $this->assertEquals([28.123], $video->lat); + $this->assertEquals([-1500], $video->metadata['gps_altitude']); + $this->assertEquals([10], $video->metadata['distance_to_ground']); + $this->assertEquals([2.6], $video->metadata['area']); + $this->assertEquals([180], $video->metadata['yaw']); + } + + public function testHandleVideoUpdate() + { + $volume = Volume::factory()->create([ + 'media_type_id' => MediaType::videoId(), + 'metadata_file_path' => 'mymeta.csv', + 'metadata_parser' => VideoCsvParser::class, + ]); + + $video = Video::factory()->create([ + 'filename' => 'a.mp4', + 'volume_id' => $volume->id, + 'taken_at' => ['2024-03-12 11:23:00'], + 'lng' => [12], + 'lat' => [34], + 'attrs' => [ + 'size' => 100, + 'metadata' => [ + 'gps_altitude' => [-1000], + 'distance_to_ground' => [5], + 'area' => [2.5], + 'yaw' => [100], + ], + ], + ]); + + + $disk = Storage::fake('metadata'); + $disk->put($volume->metadata_file_path, <<handle(); + $video->refresh(); + $this->assertEquals(100, $video->size); + $this->assertEquals([Carbon::parse('2016-12-19 12:27:00')], $video->taken_at); + $this->assertEquals([52.220], $video->lng); + $this->assertEquals([28.123], $video->lat); + $this->assertEquals([-1500], $video->metadata['gps_altitude']); + $this->assertEquals([10], $video->metadata['distance_to_ground']); + $this->assertEquals([2.6], $video->metadata['area']); + $this->assertEquals([180], $video->metadata['yaw']); + } + + public function testHandleVideoMergeWithoutTakenAt() + { + $volume = Volume::factory()->create([ + 'media_type_id' => MediaType::videoId(), + 'metadata_file_path' => 'mymeta.csv', + 'metadata_parser' => VideoCsvParser::class, + ]); + + $video = Video::factory()->create([ + 'filename' => 'a.mp4', + 'volume_id' => $volume->id, + 'lng' => [12], + 'lat' => [34], + 'attrs' => [ + 'size' => 100, + 'metadata' => [ + 'gps_altitude' => [-1000], + 'distance_to_ground' => [5], + ], + ], + ]); + + + $disk = Storage::fake('metadata'); + $disk->put($volume->metadata_file_path, <<handle(); + $video->refresh(); + $this->assertEquals(100, $video->size); + $this->assertNull($video->taken_at); + $this->assertEquals([12], $video->lng); + $this->assertEquals([34], $video->lat); + $this->assertEquals([-1500], $video->metadata['gps_altitude']); + $this->assertEquals([5], $video->metadata['distance_to_ground']); + $this->assertArrayNotHasKey('area', $video->metadata); + $this->assertArrayNotHasKey('yaw', $video->metadata); + } + + public function testHandleVideoReplaceWithTakenAt() + { + $volume = Volume::factory()->create([ + 'media_type_id' => MediaType::videoId(), + 'metadata_file_path' => 'mymeta.csv', + 'metadata_parser' => VideoCsvParser::class, + ]); + + $video = Video::factory()->create([ + 'filename' => 'a.mp4', + 'volume_id' => $volume->id, + 'lng' => [12], + 'lat' => [34], + 'attrs' => [ + 'size' => 100, + 'metadata' => [ + 'gps_altitude' => [-1000], + 'distance_to_ground' => [5], + ], + ], + ]); + + + $disk = Storage::fake('metadata'); + $disk->put($volume->metadata_file_path, <<handle(); + $video->refresh(); + $this->assertEquals(100, $video->size); + $this->assertEquals([Carbon::parse('2016-12-19 12:27:00')], $video->taken_at); + $this->assertNull($video->lng); + $this->assertNull($video->lat); + $this->assertEquals([-1500], $video->metadata['gps_altitude']); + $this->assertArrayNotHasKey('distance_to_ground', $video->metadata); + $this->assertArrayNotHasKey('area', $video->metadata); + $this->assertArrayNotHasKey('yaw', $video->metadata); + } +} diff --git a/tests/php/PendingVolumeTest.php b/tests/php/PendingVolumeTest.php new file mode 100644 index 000000000..c4455e8e4 --- /dev/null +++ b/tests/php/PendingVolumeTest.php @@ -0,0 +1,83 @@ +model->refresh(); //Populate default values. + $this->assertNotNull($this->model->media_type_id); + $this->assertNotNull($this->model->user_id); + $this->assertNotNull($this->model->project_id); + $this->assertNotNull($this->model->created_at); + $this->assertNotNull($this->model->updated_at); + $this->assertNull($this->model->metadata_file_path); + $this->assertNull($this->model->metadata_parser); + $this->assertNull($this->model->volume_id); + $this->assertFalse($this->model->import_annotations); + $this->assertFalse($this->model->import_file_labels); + $this->assertEquals([], $this->model->only_annotation_labels); + $this->assertEquals([], $this->model->only_file_labels); + $this->assertEquals([], $this->model->label_map); + $this->assertEquals([], $this->model->user_map); + $this->assertFalse($this->model->importing); + } + + public function testCreateOnlyOneForProject() + { + $this->expectException(UniqueConstraintViolationException::class); + PendingVolume::factory()->create([ + 'user_id' => $this->model->user_id, + 'project_id' => $this->model->project_id, + ]); + } + + public function testSaveMetadata() + { + $disk = Storage::fake('pending-metadata'); + $csv = __DIR__."/../files/image-metadata.csv"; + $file = new UploadedFile($csv, 'metadata.csv', 'text/csv', null, true); + + $this->assertFalse($this->model->hasMetadata()); + $this->model->saveMetadata($file); + + $disk->assertExists($this->model->id.'.csv'); + $this->assertTrue($this->model->hasMetadata()); + $this->assertEquals($this->model->id.'.csv', $this->model->metadata_file_path); + } + + public function testDeleteMetadataOnDelete() + { + $disk = Storage::fake('pending-metadata'); + $disk->put($this->model->id.'.csv', 'abc'); + $this->model->metadata_file_path = $this->model->id.'.csv'; + $this->model->save(); + $this->model->delete(); + $disk->assertMissing($this->model->id.'.csv'); + } + + public function testGetMetadata() + { + $this->assertNull($this->model->getMetadata()); + $disk = Storage::fake('pending-metadata'); + $this->model->metadata_file_path = $this->model->id.'.csv'; + $disk->put($this->model->metadata_file_path, "filename,area\n1.jpg,2.5"); + $this->model->metadata_parser = ImageCsvParser::class; + $metadata = $this->model->getMetadata(); + $fileMeta = $metadata->getFile('1.jpg'); + $this->assertEquals(2.5, $fileMeta->area); + } +} diff --git a/tests/php/Policies/PendingVolumePolicyTest.php b/tests/php/Policies/PendingVolumePolicyTest.php new file mode 100644 index 000000000..0d762c5de --- /dev/null +++ b/tests/php/Policies/PendingVolumePolicyTest.php @@ -0,0 +1,68 @@ +create(); + $this->user = User::factory()->create(); + $this->guest = User::factory()->create(); + $this->editor = User::factory()->create(); + $this->expert = User::factory()->create(); + $this->admin = User::factory()->create(); + $this->owner = User::factory()->create(); + $this->globalAdmin = User::factory()->create(['role_id' => Role::adminId()]); + $this->pv = PendingVolume::factory()->create([ + 'project_id' => $project->id, + 'user_id' => $this->owner->id, + ]); + + $project->addUserId($this->guest->id, Role::guestId()); + $project->addUserId($this->editor->id, Role::editorId()); + $project->addUserId($this->expert->id, Role::expertId()); + $project->addUserId($this->admin->id, Role::adminId()); + $project->addUserId($this->owner->id, Role::adminId()); + } + + public function testAccess() + { + $this->assertFalse($this->user->can('access', $this->pv)); + $this->assertFalse($this->guest->can('access', $this->pv)); + $this->assertFalse($this->editor->can('access', $this->pv)); + $this->assertFalse($this->expert->can('access', $this->pv)); + $this->assertFalse($this->admin->can('access', $this->pv)); + $this->assertTrue($this->owner->can('access', $this->pv)); + $this->assertTrue($this->globalAdmin->can('access', $this->pv)); + } + + public function testUpdate() + { + $this->assertFalse($this->user->can('update', $this->pv)); + $this->assertFalse($this->guest->can('update', $this->pv)); + $this->assertFalse($this->editor->can('update', $this->pv)); + $this->assertFalse($this->expert->can('update', $this->pv)); + $this->assertFalse($this->admin->can('update', $this->pv)); + $this->assertTrue($this->owner->can('update', $this->pv)); + $this->assertTrue($this->globalAdmin->can('update', $this->pv)); + } + + public function testDestroy() + { + $this->assertFalse($this->user->can('destroy', $this->pv)); + $this->assertFalse($this->guest->can('destroy', $this->pv)); + $this->assertFalse($this->editor->can('destroy', $this->pv)); + $this->assertFalse($this->expert->can('destroy', $this->pv)); + $this->assertFalse($this->admin->can('destroy', $this->pv)); + $this->assertTrue($this->owner->can('destroy', $this->pv)); + $this->assertTrue($this->globalAdmin->can('destroy', $this->pv)); + } +} diff --git a/tests/php/ProjectTest.php b/tests/php/ProjectTest.php index 22785ffa6..4abf01f87 100644 --- a/tests/php/ProjectTest.php +++ b/tests/php/ProjectTest.php @@ -4,6 +4,7 @@ use Biigle\Jobs\DeleteVolume; use Biigle\MediaType; +use Biigle\PendingVolume; use Biigle\Project; use Biigle\ProjectInvitation; use Biigle\Role; @@ -339,4 +340,11 @@ public function testInvitations() ProjectInvitation::factory(['project_id' => $this->model->id])->create(); $this->assertTrue($this->model->invitations()->exists()); } + + public function testPendingVolumes() + { + $this->assertFalse($this->model->pendingVolumes()->exists()); + PendingVolume::factory(['project_id' => $this->model->id])->create(); + $this->assertTrue($this->model->pendingVolumes()->exists()); + } } diff --git a/tests/php/Rules/ImageMetadataTest.php b/tests/php/Rules/ImageMetadataTest.php index d2c935a35..9ee4d5410 100644 --- a/tests/php/Rules/ImageMetadataTest.php +++ b/tests/php/Rules/ImageMetadataTest.php @@ -2,150 +2,101 @@ namespace Biigle\Tests\Rules; -use Biigle\Rules\ImageMetadata; +use Biigle\Rules\ImageMetadata as ImageMetadataRule; +use Biigle\Services\MetadataParsing\ImageMetadata; +use Biigle\Services\MetadataParsing\VolumeMetadata; use TestCase; class ImageMetadataTest extends TestCase { - protected static $ruleClass = ImageMetadata::class; - public function testMetadataOk() { - $validator = new static::$ruleClass(['abc.jpg']); - $metadata = [ - ['filename', 'taken_at', 'lng', 'lat', 'gps_altitude', 'distance_to_ground', 'area', 'yaw'], - ['abc.jpg', '2016-12-19 12:27:00', '52.220', '28.123', '-1500', '10', '2.6', '180'], - ]; + $validator = new ImageMetadataRule(); + + $metadata = new VolumeMetadata; + $metadata->addFile(new ImageMetadata( + name: 'abc.jpg', + takenAt: '2016-12-19 12:27:00', + lng: 52.220, + lat: 28.123, + gpsAltitude: -1500, + distanceToGround: 10, + area: 2.6, + yaw: 180 + )); $this->assertTrue($validator->passes(null, $metadata)); } - public function testMetadataWrongFile() - { - $validator = new static::$ruleClass(['abc.jpg']); - $metadata = [ - ['filename', 'taken_at'], - ['cba.jpg', '2016-12-19 12:27:00'], - ]; - $this->assertFalse($validator->passes(null, $metadata)); - } - - public function testMetadataNoCols() - { - $validator = new static::$ruleClass(['abc.jpg']); - $metadata = [ - ['abc.jpg', '2016-12-19 12:27:00'], - ]; - $this->assertFalse($validator->passes(null, $metadata)); - } - - public function testMetadataWrongCols() - { - $validator = new static::$ruleClass(['abc.jpg']); - $metadata = [ - ['filename', 'abc'], - ['abc.jpg', '2016-12-19 12:27:00'], - ]; - $this->assertFalse($validator->passes(null, $metadata)); - } - - public function testMetadataColCount() - { - $validator = new static::$ruleClass(['abc.jpg']); - $metadata = [ - ['filename', 'taken_at'], - ['abc.jpg', '2016-12-19 12:27:00', '52.220', '28.123'], - ]; - $this->assertFalse($validator->passes(null, $metadata)); - } - public function testMetadataNoLat() { - $validator = new static::$ruleClass(['abc.jpg']); - $metadata = [ - ['filename', 'lng'], - ['abc.jpg', '52.220'], - ]; + $validator = new ImageMetadataRule(); + $metadata = new VolumeMetadata; + $metadata->addFile(new ImageMetadata( + name: 'abc.jpg', + lng: 52.220 + )); $this->assertFalse($validator->passes(null, $metadata)); } public function testMetadataNoLng() { - $validator = new static::$ruleClass(['abc.jpg']); - $metadata = [ - ['filename', 'lat'], - ['abc.jpg', '28.123'], - ]; - $this->assertFalse($validator->passes(null, $metadata)); - } - - public function testMetadataColOrdering() - { - $validator = new static::$ruleClass(['abc.jpg']); - $metadata = [ - ['filename', 'lng', 'lat', 'taken_at'], - ['abc.jpg', '2016-12-19 12:27:00', '52.220', '28.123'], - ]; + $validator = new ImageMetadataRule(); + $metadata = new VolumeMetadata; + $metadata->addFile(new ImageMetadata( + name: 'abc.jpg', + lat: 28.123 + )); $this->assertFalse($validator->passes(null, $metadata)); } public function testMetadataInvalidLat() { - $validator = new static::$ruleClass(['abc.jpg']); - $metadata = [ - ['filename', 'lng', 'lat'], - ['abc.jpg', '50', '91'], - ]; + $validator = new ImageMetadataRule(); + $metadata = new VolumeMetadata; + $metadata->addFile(new ImageMetadata( + name: 'abc.jpg', + lng: 50, + lat: 91 + )); $this->assertFalse($validator->passes(null, $metadata)); } public function testMetadataInvalidLng() { - $validator = new static::$ruleClass(['abc.jpg']); - $metadata = [ - ['filename', 'lng', 'lat'], - ['abc.jpg', '181', '50'], - ]; + $validator = new ImageMetadataRule(); + $metadata = new VolumeMetadata; + $metadata->addFile(new ImageMetadata( + name: 'abc.jpg', + lng: 181, + lat: 50 + )); $this->assertFalse($validator->passes(null, $metadata)); } public function testMetadataInvalidYaw() { - $validator = new static::$ruleClass(['abc.jpg']); - $metadata = [ - ['filename', 'yaw'], - ['abc.jpg', '361'], - ]; + $validator = new ImageMetadataRule(); + $metadata = new VolumeMetadata; + $metadata->addFile(new ImageMetadata( + name: 'abc.jpg', + yaw: 361 + )); $this->assertFalse($validator->passes(null, $metadata)); } - public function testMetadataOnlyValidateFilled() - { - $validator = new static::$ruleClass(['abc.jpg']); - $metadata = [ - ['filename', 'taken_at'], - ['abc.jpg', ''], - ]; - $this->assertTrue($validator->passes(null, $metadata)); - } - - public function testMetadataLatFilledLonNotFilled() + public function testEmptyFilename() { - $validator = new static::$ruleClass(['abc.jpg']); - $metadata = [ - ['filename', 'lat', 'lon'], - ['abc.jpg', '28.123', ''], - ]; + $validator = new ImageMetadataRule(); + $metadata = new VolumeMetadata; + $metadata->addFile(new ImageMetadata(name: '')); $this->assertFalse($validator->passes(null, $metadata)); } - public function testEmptyFilename() + public function testEmpty() { - $validator = new static::$ruleClass(['abc.jpg']); - $metadata = [ - ['filename', 'taken_at'], - ['abc.jpg', ''], - ['', ''], - ]; + $validator = new ImageMetadataRule(); + $metadata = new VolumeMetadata; + $metadata->addFile(new ImageMetadata(name: 'abc.jpg')); $this->assertFalse($validator->passes(null, $metadata)); } } diff --git a/tests/php/Rules/VideoMetadataTest.php b/tests/php/Rules/VideoMetadataTest.php index f44c4eb40..017658a44 100644 --- a/tests/php/Rules/VideoMetadataTest.php +++ b/tests/php/Rules/VideoMetadataTest.php @@ -2,52 +2,171 @@ namespace Biigle\Tests\Rules; -use Biigle\Rules\VideoMetadata; +use Biigle\Rules\VideoMetadata as VideoMetadataRule; +use Biigle\Services\MetadataParsing\VideoMetadata; +use Biigle\Services\MetadataParsing\VolumeMetadata; +use TestCase; -class VideoMetadataTest extends ImageMetadataTest +class VideoMetadataTest extends TestCase { - protected static $ruleClass = VideoMetadata::class; - - public function testMultipleRowsWithTakenAtCol() + public function testMetadataOk() { - $validator = new static::$ruleClass(['abc.mp4']); - $metadata = [ - ['filename', 'taken_at', 'lng', 'lat'], - ['abc.mp4', '2016-12-19 12:27:00', '52.220', '28.123'], - ['abc.mp4', '2016-12-19 12:28:00', '52.230', '28.133'], - ]; + $validator = new VideoMetadataRule(); + + $metadata = new VolumeMetadata; + $metadata->addFile(new VideoMetadata( + name: 'abc.mp4', + takenAt: '2016-12-19 12:27:00', + lng: 52.220, + lat: 28.123, + gpsAltitude: -1500, + distanceToGround: 10, + area: 2.6, + yaw: 180 + )); $this->assertTrue($validator->passes(null, $metadata)); } - public function testMultipleRowsWithoutTakenAtCol() + public function testMetadataNoLat() + { + $validator = new VideoMetadataRule(); + $metadata = new VolumeMetadata; + $metadata->addFile(new VideoMetadata( + name: 'abc.mp4', + lng: 52.220 + )); + $this->assertFalse($validator->passes(null, $metadata)); + } + + public function testMetadataNoLatFrame() + { + $validator = new VideoMetadataRule(); + $metadata = new VolumeMetadata; + $fileMeta = new VideoMetadata(name: 'abc.mp4'); + $fileMeta->addFrame('2016-12-19 12:27:00', lng: 52.220); + $metadata->addFile($fileMeta); + $this->assertFalse($validator->passes(null, $metadata)); + } + + public function testMetadataNoLng() + { + $validator = new VideoMetadataRule(); + $metadata = new VolumeMetadata; + $metadata->addFile(new VideoMetadata( + name: 'abc.mp4', + lat: 28.123 + )); + $this->assertFalse($validator->passes(null, $metadata)); + } + + public function testMetadataNoLngFrame() + { + $validator = new VideoMetadataRule(); + $metadata = new VolumeMetadata; + $fileMeta = new VideoMetadata(name: 'abc.mp4'); + $fileMeta->addFrame('2016-12-19 12:27:00', lat: 28.123); + $metadata->addFile($fileMeta); + $this->assertFalse($validator->passes(null, $metadata)); + } + + public function testMetadataInvalidLat() + { + $validator = new VideoMetadataRule(); + $metadata = new VolumeMetadata; + $metadata->addFile(new VideoMetadata( + name: 'abc.mp4', + lng: 50, + lat: 91 + )); + $this->assertFalse($validator->passes(null, $metadata)); + } + + public function testMetadataInvalidLatFrame() { - $validator = new static::$ruleClass(['abc.mp4']); - $metadata = [ - ['filename', 'lng', 'lat'], - ['abc.mp4', '52.220', '28.123'], - ['abc.mp4', '52.230', '28.133'], - ]; + $validator = new VideoMetadataRule(); + $metadata = new VolumeMetadata; + $fileMeta = new VideoMetadata(name: 'abc.mp4'); + $fileMeta->addFrame('2016-12-19 12:27:00', lng: 50, lat: 91); + $metadata->addFile($fileMeta); $this->assertFalse($validator->passes(null, $metadata)); } - public function testMultipleRowsWithEmptyTakenAtCol() + public function testMetadataInvalidLng() { - $validator = new static::$ruleClass(['abc.mp4']); - $metadata = [ - ['filename', 'taken_at', 'lng', 'lat'], - ['abc.mp4', '2016-12-19 12:27:00', '52.220', '28.123'], - ['abc.mp4', '', '52.230', '28.133'], - ]; + $validator = new VideoMetadataRule(); + $metadata = new VolumeMetadata; + $metadata->addFile(new VideoMetadata( + name: 'abc.mp4', + lng: 181, + lat: 50 + )); $this->assertFalse($validator->passes(null, $metadata)); } - public function testOneRowWithoutTakenAtCol() + public function testMetadataInvalidLngFrame() { - $validator = new static::$ruleClass(['abc.mp4']); - $metadata = [ - ['filename', 'lng', 'lat'], - ['abc.mp4', '52.220', '28.123'], - ]; + $validator = new VideoMetadataRule(); + $metadata = new VolumeMetadata; + $fileMeta = new VideoMetadata(name: 'abc.mp4'); + $fileMeta->addFrame('2016-12-19 12:27:00', lng: 181, lat: 50); + $metadata->addFile($fileMeta); + $this->assertFalse($validator->passes(null, $metadata)); + } + + public function testMetadataInvalidYaw() + { + $validator = new VideoMetadataRule(); + $metadata = new VolumeMetadata; + $metadata->addFile(new VideoMetadata( + name: 'abc.mp4', + yaw: 361 + )); + $this->assertFalse($validator->passes(null, $metadata)); + } + + public function testMetadataInvalidYawFrame() + { + $validator = new VideoMetadataRule(); + $metadata = new VolumeMetadata; + $fileMeta = new VideoMetadata(name: 'abc.mp4'); + $fileMeta->addFrame('2016-12-19 12:27:00', yaw: 361); + $metadata->addFile($fileMeta); + $this->assertFalse($validator->passes(null, $metadata)); + } + + public function testEmptyFilename() + { + $validator = new VideoMetadataRule(); + $metadata = new VolumeMetadata; + $metadata->addFile(new VideoMetadata(name: '')); + $this->assertFalse($validator->passes(null, $metadata)); + } + + public function testEmpty() + { + $validator = new VideoMetadataRule(['abc.jpg']); + $metadata = new VolumeMetadata; + $metadata->addFile(new VideoMetadata(name: 'abc.jpg')); + $this->assertFalse($validator->passes(null, $metadata)); + } + + public function testMultipleFrames() + { + $validator = new VideoMetadataRule(); + + $metadata = new VolumeMetadata; + $fileMeta = new VideoMetadata( + name: 'abc.mp4', + takenAt: '2016-12-19 12:27:00', + lng: 52.220, + lat: 28.123 + ); + $fileMeta->addFrame( + takenAt: '2016-12-19 12:28:00', + lng: 52.230, + lat: 28.133 + ); + $metadata->addFile($fileMeta); $this->assertTrue($validator->passes(null, $metadata)); } } diff --git a/tests/php/Services/MetadataParsing/FileMetadataTest.php b/tests/php/Services/MetadataParsing/FileMetadataTest.php new file mode 100644 index 000000000..c019c6054 --- /dev/null +++ b/tests/php/Services/MetadataParsing/FileMetadataTest.php @@ -0,0 +1,167 @@ +assertEquals('filename', $data->name); + } + + public function testIsEmpty() + { + $data = new ImageMetadata('filename'); + $this->assertTrue($data->isEmpty()); + + $data = new ImageMetadata('filename', area: 10); + $this->assertFalse($data->isEmpty()); + } + + public function testHasAnnotations() + { + $data = new ImageMetadata('filename'); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $annotation = new ImageAnnotation( + shape: Shape::point(), + points: [10, 10], + labels: [$lau], + ); + + $this->assertFalse($data->hasAnnotations()); + $data->addAnnotation($annotation); + $this->assertTrue($data->hasAnnotations()); + } + + public function testGetFileLabelLabels() + { + $data = new ImageMetadata('filename'); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + + $this->assertEquals([], $data->getFileLabelLabels()); + $data->addFileLabel($lau); + $this->assertEquals([123 => $label], $data->getFileLabelLabels()); + + $label2 = new Label(456, 'my label'); + $lau = new LabelAndUser($label2, $user); + $this->assertEquals([123 => $label], $data->getFileLabelLabels([123])); + } + + public function testHasFileLabels() + { + $data = new ImageMetadata('filename'); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + + $this->assertFalse($data->hasFileLabels()); + $data->addFileLabel($lau); + $this->assertTrue($data->hasFileLabels()); + } + + public function testGetAnnotationLabels() + { + $data = new ImageMetadata('filename'); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user); + $annotation = new ImageAnnotation( + shape: Shape::point(), + points: [10, 10], + labels: [$lau], + ); + + $this->assertEquals([], $data->getAnnotationLabels()); + $data->addAnnotation($annotation); + $this->assertEquals([123 => $label], $data->getAnnotationLabels()); + + $label2 = new Label(456, 'my label'); + $lau = new LabelAndUser($label2, $user); + $annotation2 = new ImageAnnotation( + shape: Shape::point(), + points: [10, 10], + labels: [$lau], + ); + $data->addAnnotation($annotation2); + $this->assertEquals([123 => $label], $data->getAnnotationLabels([123])); + } + + public function testGetUsers() + { + $data = new ImageMetadata('filename'); + $label = new Label(123, 'my label'); + $user1 = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user1); + $annotation = new ImageAnnotation( + shape: Shape::point(), + points: [10, 10], + labels: [$lau], + ); + + $this->assertEquals([], $data->getUsers()); + $data->addAnnotation($annotation); + $this->assertEquals([321 => $user1], $data->getUsers()); + + $user2 = new User(432, 'joe user'); + $lau = new LabelAndUser($label, $user2); + $data->addFileLabel($lau); + $this->assertEquals([321 => $user1, 432 => $user2], $data->getUsers()); + } + + public function testGetFileLabelUsersOnlyLabels() + { + $data = new ImageMetadata('filename'); + + $label1 = new Label(123, 'my label'); + $user1 = new User(432, 'joe user'); + $lau = new LabelAndUser($label1, $user1); + $data->addFileLabel($lau); + + $label2 = new Label(456, 'my label'); + $user2 = new User(654, 'joe user'); + $lau = new LabelAndUser($label2, $user2); + $data->addFileLabel($lau); + + $this->assertEquals([432 => $user1], $data->getUsers([123])); + } + + public function testGetAnnotationUsersOnlyLabels() + { + $data = new ImageMetadata('filename'); + + $label1 = new Label(123, 'my label'); + $user1 = new User(432, 'joe user'); + $lau = new LabelAndUser($label1, $user1); + $annotation = new ImageAnnotation( + shape: Shape::point(), + points: [10, 10], + labels: [$lau], + ); + $data->addAnnotation($annotation); + + $label2 = new Label(456, 'my label'); + $user2 = new User(654, 'joe user'); + $lau = new LabelAndUser($label2, $user2); + $annotation = new ImageAnnotation( + shape: Shape::point(), + points: [10, 10], + labels: [$lau], + ); + $data->addAnnotation($annotation); + + $this->assertEquals([432 => $user1], $data->getUsers([123])); + } +} diff --git a/tests/php/Services/MetadataParsing/ImageAnnotationTest.php b/tests/php/Services/MetadataParsing/ImageAnnotationTest.php new file mode 100644 index 000000000..24a95605f --- /dev/null +++ b/tests/php/Services/MetadataParsing/ImageAnnotationTest.php @@ -0,0 +1,55 @@ + 123, + 'points' => '[10,10]', + 'shape_id' => Shape::pointId(), + ]; + + $this->assertEquals($expect, $data->getInsertData(123)); + } + + public function testValidateLabels() + { + $data = new ImageAnnotation( + shape: Shape::point(), + points: [10, 10], + labels: [], + ); + + $this->expectException(Exception::class); + $data->validate(); + } + + public function testValidatePoints() + { + $data = new ImageAnnotation( + shape: Shape::point(), + points: [10, 10, 10], + labels: [new LabelAndUser(new Label(1, 'x'), new User(2, 'y'))], + ); + + $this->expectException(Exception::class); + $data->validate(); + } +} diff --git a/tests/php/Services/MetadataParsing/ImageCsvParserTest.php b/tests/php/Services/MetadataParsing/ImageCsvParserTest.php new file mode 100644 index 000000000..d324181c6 --- /dev/null +++ b/tests/php/Services/MetadataParsing/ImageCsvParserTest.php @@ -0,0 +1,182 @@ +assertTrue($parser->recognizesFile()); + + $file = new File(__DIR__."/../../../files/image-metadata-with-bom.csv"); + $parser = new ImageCsvParser($file); + $this->assertTrue($parser->recognizesFile()); + + $file = new File(__DIR__."/../../../files/test.mp4"); + $parser = new ImageCsvParser($file); + $this->assertFalse($parser->recognizesFile()); + + $file = new File(__DIR__."/../../../files/image-metadata-strange-encoding.csv"); + $parser = new ImageCsvParser($file); + $this->assertFalse($parser->recognizesFile()); + } + + public function testGetMetadata() + { + $file = new File(__DIR__."/../../../files/image-metadata.csv"); + $parser = new ImageCsvParser($file); + $data = $parser->getMetadata(); + $this->assertEquals(MediaType::imageId(), $data->type->id); + $this->assertNull($data->name); + $this->assertNull($data->url); + $this->assertNull($data->handle); + $this->assertCount(1, $data->getFiles()); + $file = $data->getFiles()->first(); + $this->assertEquals('abc.jpg', $file->name); + $this->assertEquals('2016-12-19 12:27:00', $file->takenAt); + $this->assertEquals(52.220, $file->lng); + $this->assertEquals(28.123, $file->lat); + $this->assertEquals(-1500, $file->gpsAltitude); + $this->assertEquals(10, $file->distanceToGround); + $this->assertEquals(2.6, $file->area); + $this->assertEquals(180, $file->yaw); + } + + public function testGetMetadataIgnoreMissingFilename() + { + $file = new File(__DIR__."/../../../files/image-metadata.csv"); + $parser = new ImageCsvParserStub($file); + $parser->content = [ + ['filename', 'taken_at', 'lng', 'lat', 'gps_altitude', 'distance_to_ground', 'area', 'yaw'], + ['', '2016-12-19 12:27:00', '52.220', '28.123', '-1500', '10', '2.6', '180'], + ]; + + $data = $parser->getMetadata(); + $this->assertCount(0, $data->getFiles()); + } + + public function testGetMetadataCantReadFile() + { + $file = new File(__DIR__."/../../../files/test.mp4"); + $parser = new ImageCsvParser($file); + $data = $parser->getMetadata(); + $this->assertEquals(MediaType::imageId(), $data->type->id); + $this->assertCount(0, $data->getFiles()); + } + + public function testGetMetadataCaseInsensitive() + { + $file = new File(__DIR__."/../../../files/image-metadata.csv"); + $parser = new ImageCsvParserStub($file); + $parser->content = [ + ['Filename', 'tAken_at', 'lnG', 'Lat', 'gPs_altitude', 'diStance_to_ground', 'areA'], + ['abc.jpg', '2016-12-19 12:27:00', '52.220', '28.123', '-1500', '10', '2.6'], + ]; + + $data = $parser->getMetadata(); + $this->assertCount(1, $data->getFiles()); + $file = $data->getFiles()->first(); + $this->assertEquals('abc.jpg', $file->name); + $this->assertEquals('2016-12-19 12:27:00', $file->takenAt); + $this->assertEquals(52.220, $file->lng); + $this->assertEquals(28.123, $file->lat); + $this->assertEquals(-1500, $file->gpsAltitude); + $this->assertEquals(10, $file->distanceToGround); + $this->assertEquals(2.6, $file->area); + } + + public function testGetMetadataColumnSynonyms1() + { + $file = new File(__DIR__."/../../../files/image-metadata.csv"); + $parser = new ImageCsvParserStub($file); + $parser->content = [ + ['file', 'lon', 'lat', 'heading'], + ['abc.jpg', '52.220', '28.123', '180'], + ]; + + $data = $parser->getMetadata(); + $this->assertCount(1, $data->getFiles()); + $file = $data->getFiles()->first(); + $this->assertEquals('abc.jpg', $file->name); + $this->assertEquals(52.220, $file->lng); + $this->assertEquals(28.123, $file->lat); + $this->assertEquals(180, $file->yaw); + } + + public function testGetMetadataColumnSynonyms2() + { + $file = new File(__DIR__."/../../../files/image-metadata.csv"); + $parser = new ImageCsvParserStub($file); + $parser->content = [ + ['file', 'longitude', 'latitude'], + ['abc.jpg', '52.220', '28.123'], + ]; + + $data = $parser->getMetadata(); + $this->assertCount(1, $data->getFiles()); + $file = $data->getFiles()->first(); + $this->assertEquals('abc.jpg', $file->name); + $this->assertEquals(52.220, $file->lng); + $this->assertEquals(28.123, $file->lat); + } + + public function testGetMetadataColumnSynonyms3() + { + $file = new File(__DIR__."/../../../files/image-metadata.csv"); + $parser = new ImageCsvParserStub($file); + $parser->content = [ + ['file', 'SUB_datetime', 'SUB_longitude', 'SUB_latitude', 'SUB_altitude', 'SUB_distance', 'SUB_heading'], + ['abc.jpg', '2016-12-19 12:27:00', '52.220', '28.123', '-1500', '10', '180'], + ]; + + $data = $parser->getMetadata(); + $this->assertCount(1, $data->getFiles()); + $file = $data->getFiles()->first(); + $this->assertEquals('abc.jpg', $file->name); + $this->assertEquals('2016-12-19 12:27:00', $file->takenAt); + $this->assertEquals(52.220, $file->lng); + $this->assertEquals(28.123, $file->lat); + $this->assertEquals(-1500, $file->gpsAltitude); + $this->assertEquals(10, $file->distanceToGround); + $this->assertEquals(180, $file->yaw); + } + + public function testGetMetadataEmptyCells() + { + $file = new File(__DIR__."/../../../files/image-metadata.csv"); + $parser = new ImageCsvParserStub($file); + $parser->content = [ + ['filename', 'taken_at', 'lng', 'lat', 'gps_altitude', 'distance_to_ground', 'area', 'yaw'], + ['abc.jpg', '', '52.220', '28.123', '', '', '', ''], + ]; + + $data = $parser->getMetadata(); + $this->assertCount(1, $data->getFiles()); + $file = $data->getFiles()->first(); + $this->assertEquals('abc.jpg', $file->name); + $this->assertNull($file->takenAt); + $this->assertEquals(52.220, $file->lng); + $this->assertEquals(28.123, $file->lat); + $this->assertNull($file->gpsAltitude); + $this->assertNull($file->distanceToGround); + $this->assertNull($file->area); + $this->assertNull($file->yaw); + } +} + +class ImageCsvParserStub extends ImageCsvParser +{ + public array $content = []; + + protected function getCsvIterator(): \SeekableIterator + { + return new \ArrayIterator($this->content); + } +} diff --git a/tests/php/Services/MetadataParsing/ImageMetadataTest.php b/tests/php/Services/MetadataParsing/ImageMetadataTest.php new file mode 100644 index 000000000..04e2386ef --- /dev/null +++ b/tests/php/Services/MetadataParsing/ImageMetadataTest.php @@ -0,0 +1,40 @@ + '1.jpg', + 'lat' => 100, + 'lng' => 120, + 'taken_at' => '2024-03-11 16:43:00', + 'attrs' => [ + 'metadata' => [ + 'area' => 2.5, + 'distance_to_ground' => 5, + 'gps_altitude' => -1500, + 'yaw' => 50, + ], + ], + ]; + + $this->assertEquals($expect, $data->getInsertData()); + } +} diff --git a/tests/php/Services/MetadataParsing/ParserFactoryTest.php b/tests/php/Services/MetadataParsing/ParserFactoryTest.php new file mode 100644 index 000000000..de18a0aff --- /dev/null +++ b/tests/php/Services/MetadataParsing/ParserFactoryTest.php @@ -0,0 +1,70 @@ +assertTrue(ParserFactory::has('image', ImageCsvParser::class)); + } + + public function testHasVideo() + { + $this->assertTrue(ParserFactory::has('video', VideoCsvParser::class)); + } + + public function testHasUnknownType() + { + $this->assertFalse(ParserFactory::has('unknown', ImageCsvParser::class)); + } + + public function testHasUnknownParser() + { + $this->assertFalse(ParserFactory::has('image', 'unknown')); + } + + public function testExtend() + { + ParserFactory::extend(TestParser::class, 'image'); + $this->assertContains(TestParser::class, ParserFactory::$parsers['image']); + + $this->expectException(\Exception::class); + ParserFactory::extend(TestParser2::class, 'image'); + } +} + +class TestParser extends MetadataParser +{ + public static function getKnownMimeTypes(): array + { + return []; + } + + public static function getName(): string + { + return 'Test'; + } + + public function recognizesFile(): bool + { + return false; + } + + public function getMetadata(): VolumeMetadata + { + return new VolumeMetadata; + } +} + +class TestParser2 +{ + // +} diff --git a/tests/php/Services/MetadataParsing/VideoAnnotationTest.php b/tests/php/Services/MetadataParsing/VideoAnnotationTest.php new file mode 100644 index 000000000..9d724bff2 --- /dev/null +++ b/tests/php/Services/MetadataParsing/VideoAnnotationTest.php @@ -0,0 +1,98 @@ + 123, + 'points' => '[[10,10]]', + 'shape_id' => Shape::pointId(), + 'frames' => '[1]', + ]; + + $this->assertEquals($expect, $data->getInsertData(123)); + } + + public function testValidateLabels() + { + $data = new VideoAnnotation( + shape: Shape::point(), + points: [[10, 10]], + frames: [1], + labels: [], + ); + + $this->expectException(Exception::class); + $data->validate(); + } + + public function testValidatePoints() + { + $data = new VideoAnnotation( + shape: Shape::point(), + points: [[10, 10, 10]], + frames: [1], + labels: [new LabelAndUser(new Label(1, 'x'), new User(2, 'y'))], + ); + + $this->expectException(Exception::class); + $data->validate(); + } + + public function testValidatePointsArray() + { + $data = new VideoAnnotation( + shape: Shape::point(), + points: [10, 10], + frames: [1], + labels: [new LabelAndUser(new Label(1, 'x'), new User(2, 'y'))], + ); + + $this->expectException(Exception::class); + $data->validate(); + } + + public function testValidateFramesArray1() + { + $data = new VideoAnnotation( + shape: Shape::point(), + points: [[10, 10]], + frames: [], + labels: [new LabelAndUser(new Label(1, 'x'), new User(2, 'y'))], + ); + + $this->expectException(Exception::class); + $data->validate(); + } + + public function testValidateFramesArray2() + { + $data = new VideoAnnotation( + shape: Shape::point(), + points: [[10, 10]], + frames: ['a'], + labels: [new LabelAndUser(new Label(1, 'x'), new User(2, 'y'))], + ); + + $this->expectException(Exception::class); + $data->validate(); + } +} diff --git a/tests/php/Services/MetadataParsing/VideoCsvParserTest.php b/tests/php/Services/MetadataParsing/VideoCsvParserTest.php new file mode 100644 index 000000000..2b56ae861 --- /dev/null +++ b/tests/php/Services/MetadataParsing/VideoCsvParserTest.php @@ -0,0 +1,214 @@ +assertTrue($parser->recognizesFile()); + + $file = new File(__DIR__."/../../../files/test.mp4"); + $parser = new VideoCsvParser($file); + $this->assertFalse($parser->recognizesFile()); + + $file = new File(__DIR__."/../../../files/video-metadata-strange-encoding.csv"); + $parser = new VideoCsvParser($file); + $this->assertFalse($parser->recognizesFile()); + } + + public function testGetMetadata() + { + $file = new File(__DIR__."/../../../files/video-metadata.csv"); + $parser = new VideoCsvParser($file); + $data = $parser->getMetadata(); + $this->assertEquals(MediaType::videoId(), $data->type->id); + $this->assertNull($data->name); + $this->assertNull($data->url); + $this->assertNull($data->handle); + $this->assertCount(1, $data->getFiles()); + $file = $data->getFiles()->first(); + $this->assertEquals('abc.mp4', $file->name); + $this->assertEquals('2016-12-19 12:27:00', $file->takenAt); + $this->assertEquals(52.220, $file->lng); + $this->assertEquals(28.123, $file->lat); + $this->assertEquals(-1500, $file->gpsAltitude); + $this->assertEquals(10, $file->distanceToGround); + $this->assertEquals(2.6, $file->area); + $this->assertEquals(180, $file->yaw); + + $frames = $file->getFrames(); + $this->assertCount(2, $frames); + $frame = $frames[0]; + $this->assertEquals('abc.mp4', $frame->name); + $this->assertEquals('2016-12-19 12:27:00', $frame->takenAt); + $this->assertEquals(52.220, $frame->lng); + $this->assertEquals(28.123, $frame->lat); + $this->assertEquals(-1500, $frame->gpsAltitude); + $this->assertEquals(10, $frame->distanceToGround); + $this->assertEquals(2.6, $frame->area); + $this->assertEquals(180, $frame->yaw); + + $frame = $frames[1]; + $this->assertEquals('abc.mp4', $frame->name); + $this->assertEquals('2016-12-19 12:28:00', $frame->takenAt); + $this->assertEquals(52.230, $frame->lng); + $this->assertEquals(28.133, $frame->lat); + $this->assertEquals(-1505, $frame->gpsAltitude); + $this->assertEquals(5, $frame->distanceToGround); + $this->assertEquals(1.6, $frame->area); + $this->assertEquals(181, $frame->yaw); + } + + public function testGetMetadataIgnoreMissingFilename() + { + $file = new File(__DIR__."/../../../files/video-metadata.csv"); + $parser = new VideoCsvParserStub($file); + $parser->content = [ + ['filename', 'taken_at', 'lng', 'lat', 'gps_altitude', 'distance_to_ground', 'area', 'yaw'], + ['', '2016-12-19 12:27:00', '52.220', '28.123', '-1500', '10', '2.6', '180'], + ]; + + $data = $parser->getMetadata(); + $this->assertCount(0, $data->getFiles()); + } + + public function testGetMetadataCantReadFile() + { + $file = new File(__DIR__."/../../../files/test.mp4"); + $parser = new VideoCsvParser($file); + $data = $parser->getMetadata(); + $this->assertEquals(MediaType::videoId(), $data->type->id); + $this->assertCount(0, $data->getFiles()); + } + + public function testGetMetadataCaseInsensitive() + { + $file = new File(__DIR__."/../../../files/video-metadata.csv"); + $parser = new VideoCsvParserStub($file); + $parser->content = [ + ['Filename', 'tAken_at', 'lnG', 'Lat', 'gPs_altitude', 'diStance_to_ground', 'areA'], + ['abc.mp4', '2016-12-19 12:27:00', '52.220', '28.123', '-1500', '10', '2.6'], + ]; + + $data = $parser->getMetadata(); + $this->assertCount(1, $data->getFiles()); + $file = $data->getFiles()->first(); + $this->assertEquals('abc.mp4', $file->name); + $frame = $file->getFrames()->first(); + $this->assertEquals('2016-12-19 12:27:00', $frame->takenAt); + $this->assertEquals(52.220, $frame->lng); + $this->assertEquals(28.123, $frame->lat); + $this->assertEquals(-1500, $frame->gpsAltitude); + $this->assertEquals(10, $frame->distanceToGround); + $this->assertEquals(2.6, $frame->area); + } + + public function testGetMetadataColumnSynonyms1() + { + $file = new File(__DIR__."/../../../files/video-metadata.csv"); + $parser = new VideoCsvParserStub($file); + $parser->content = [ + ['file', 'lon', 'lat', 'heading'], + ['abc.mp4', '52.220', '28.123', '180'], + ]; + + $data = $parser->getMetadata(); + $this->assertCount(1, $data->getFiles()); + $file = $data->getFiles()->first(); + $this->assertEquals('abc.mp4', $file->name); + $this->assertEquals(52.220, $file->lng); + $this->assertEquals(28.123, $file->lat); + $this->assertEquals(180, $file->yaw); + $this->assertTrue($file->getFrames()->isEmpty()); + } + + public function testGetMetadataColumnSynonyms2() + { + $file = new File(__DIR__."/../../../files/video-metadata.csv"); + $parser = new VideoCsvParserStub($file); + $parser->content = [ + ['file', 'longitude', 'latitude'], + ['abc.mp4', '52.220', '28.123'], + ]; + + $data = $parser->getMetadata(); + $this->assertCount(1, $data->getFiles()); + $file = $data->getFiles()->first(); + $this->assertEquals('abc.mp4', $file->name); + $this->assertEquals(52.220, $file->lng); + $this->assertEquals(28.123, $file->lat); + $this->assertTrue($file->getFrames()->isEmpty()); + } + + public function testGetMetadataColumnSynonyms3() + { + $file = new File(__DIR__."/../../../files/video-metadata.csv"); + $parser = new VideoCsvParserStub($file); + $parser->content = [ + ['file', 'SUB_datetime', 'SUB_longitude', 'SUB_latitude', 'SUB_altitude', 'SUB_distance', 'SUB_heading'], + ['abc.mp4', '2016-12-19 12:27:00', '52.220', '28.123', '-1500', '10', '180'], + ]; + + $data = $parser->getMetadata(); + $this->assertCount(1, $data->getFiles()); + $file = $data->getFiles()->first(); + $this->assertEquals('abc.mp4', $file->name); + $frame = $file->getFrames()->first(); + $this->assertEquals('2016-12-19 12:27:00', $frame->takenAt); + $this->assertEquals(52.220, $frame->lng); + $this->assertEquals(28.123, $frame->lat); + $this->assertEquals(-1500, $frame->gpsAltitude); + $this->assertEquals(10, $frame->distanceToGround); + $this->assertEquals(180, $frame->yaw); + } + + public function testGetMetadataEmptyCells() + { + $file = new File(__DIR__."/../../../files/video-metadata.csv"); + $parser = new VideoCsvParserStub($file); + $parser->content = [ + ['filename', 'taken_at', 'lng', 'lat', 'gps_altitude', 'distance_to_ground', 'area', 'yaw'], + ['abc.mp4', '', '52.220', '28.123', '', '', '', ''], + ]; + + $data = $parser->getMetadata(); + $this->assertCount(1, $data->getFiles()); + $file = $data->getFiles()->first(); + $this->assertEquals('abc.mp4', $file->name); + $this->assertNull($file->takenAt); + $this->assertEquals(52.220, $file->lng); + $this->assertEquals(28.123, $file->lat); + $this->assertNull($file->gpsAltitude); + $this->assertNull($file->distanceToGround); + $this->assertNull($file->area); + $this->assertNull($file->yaw); + $this->assertTrue($file->getFrames()->isEmpty()); + } + + public function testGetMetadataStrangeEncoding() + { + $file = new File(__DIR__."/../../../files/video-metadata-strange-encoding.csv"); + $parser = new VideoCsvParser($file); + $data = $parser->getMetadata(); + $this->assertCount(1, $data->getFiles()); + $this->assertCount(0, $data->getFiles()->first()->getFrames()); + } +} + +class VideoCsvParserStub extends VideoCsvParser +{ + public array $content = []; + + protected function getCsvIterator(): \SeekableIterator + { + return new \ArrayIterator($this->content); + } +} diff --git a/tests/php/Services/MetadataParsing/VideoMetadataTest.php b/tests/php/Services/MetadataParsing/VideoMetadataTest.php new file mode 100644 index 000000000..be359e156 --- /dev/null +++ b/tests/php/Services/MetadataParsing/VideoMetadataTest.php @@ -0,0 +1,180 @@ +assertTrue($data->isEmpty()); + $data->addFrame('2023-12-12 20:26:00'); + $this->assertFalse($data->isEmpty()); + + $data = new VideoMetadata('filename', area: 10); + $this->assertFalse($data->isEmpty()); + } + + public function testGetFrames() + { + $data = new VideoMetadata('filename'); + $this->assertTrue($data->getFrames()->isEmpty()); + + $data = new VideoMetadata('filename', takenAt: '2023-12-12 20:26:00'); + $frame = $data->getFrames()->first(); + $this->assertEquals('filename', $frame->name); + $this->assertEquals('2023-12-12 20:26:00', $frame->takenAt); + } + + public function testAddFrame() + { + $data = new VideoMetadata('filename'); + $this->assertTrue($data->getFrames()->isEmpty()); + $data->addFrame('2023-12-12 20:26:00'); + $frame = $data->getFrames()->first(); + $this->assertEquals('filename', $frame->name); + $this->assertEquals('2023-12-12 20:26:00', $frame->takenAt); + } + + public function testGetInsertDataPlain() + { + $data = new VideoMetadata( + '1.mp4', + lat: 100, + lng: 120, + area: 2.5, + distanceToGround: 5, + gpsAltitude: -1500, + yaw: 50 + ); + + $expect = [ + 'filename' => '1.mp4', + 'lat' => [100], + 'lng' => [120], + 'attrs' => [ + 'metadata' => [ + 'area' => [2.5], + 'distance_to_ground' => [5], + 'gps_altitude' => [-1500], + 'yaw' => [50], + ], + ], + ]; + + $this->assertEquals($expect, $data->getInsertData()); + } + + public function testGetInsertDataFrame() + { + $data = new VideoMetadata( + '1.mp4', + lat: 100, + lng: 120, + takenAt: '03/11/2024 16:43:00', + area: 2.5, + distanceToGround: 5, + gpsAltitude: -1500, + yaw: 50 + ); + + $expect = [ + 'filename' => '1.mp4', + 'lat' => [100], + 'lng' => [120], + 'taken_at' => ['2024-03-11 16:43:00'], + 'attrs' => [ + 'metadata' => [ + 'area' => [2.5], + 'distance_to_ground' => [5], + 'gps_altitude' => [-1500], + 'yaw' => [50], + ], + ], + ]; + + $this->assertEquals($expect, $data->getInsertData()); + } + + public function testGetInsertDataFrames() + { + $data = new VideoMetadata('1.mp4'); + + $data->addFrame( + '2024-03-11 16:44:00', + lat: 110, + lng: 130, + area: 3, + distanceToGround: 4, + gpsAltitude: -1501, + yaw: 51 + ); + + $data->addFrame( + '2024-03-11 16:43:00', + lat: 100, + lng: 120, + area: 2.5, + distanceToGround: 5, + gpsAltitude: -1500, + yaw: 50 + ); + + $expect = [ + 'filename' => '1.mp4', + 'lat' => [100, 110], + 'lng' => [120, 130], + // Metadata should be sorted by taken_at. + 'taken_at' => ['2024-03-11 16:43:00', '2024-03-11 16:44:00'], + 'attrs' => [ + 'metadata' => [ + 'area' => [2.5, 3], + 'distance_to_ground' => [5, 4], + 'gps_altitude' => [-1500, -1501], + 'yaw' => [50, 51], + ], + ], + ]; + + $this->assertEquals($expect, $data->getInsertData()); + } + + public function testGetInsertDataFramesWithGaps() + { + $data = new VideoMetadata('1.mp4'); + + $data->addFrame( + '2024-03-11 16:44:00', + lat: 110, + lng: 130, + gpsAltitude: -1501 + ); + + $data->addFrame( + '2024-03-11 16:43:00', + area: 2.5, + distanceToGround: 5, + gpsAltitude: -1500 + ); + + $expect = [ + 'filename' => '1.mp4', + 'lat' => [null, 110], + 'lng' => [null, 130], + // Metadata should be sorted by taken_at. + 'taken_at' => ['2024-03-11 16:43:00', '2024-03-11 16:44:00'], + 'attrs' => [ + 'metadata' => [ + 'area' => [2.5, null], + 'distance_to_ground' => [5, null], + 'gps_altitude' => [-1500, -1501], + ], + ], + ]; + + $this->assertEquals($expect, $data->getInsertData()); + } +} diff --git a/tests/php/Services/MetadataParsing/VolumeMetadataTest.php b/tests/php/Services/MetadataParsing/VolumeMetadataTest.php new file mode 100644 index 000000000..b26ba9233 --- /dev/null +++ b/tests/php/Services/MetadataParsing/VolumeMetadataTest.php @@ -0,0 +1,373 @@ +assertEquals(MediaType::imageId(), $metadata->type->id); + $this->assertEquals('volumename', $metadata->name); + $this->assertEquals('volumeurl', $metadata->url); + $this->assertEquals('volumehandle', $metadata->handle); + } + + public function testAddGetFiles() + { + $metadata = new VolumeMetadata; + $file = new FileMetadata('filename'); + $metadata->addFile($file); + $this->assertEquals($file, $metadata->getFiles()[0]); + $metadata->addFile($file); + $this->assertCount(1, $metadata->getFiles()); + } + + public function testGetFile() + { + $metadata = new VolumeMetadata; + $this->assertNull($metadata->getFile('filename')); + $file = new FileMetadata('filename'); + $metadata->addFile($file); + $this->assertEquals($file, $metadata->getFile('filename')); + } + + public function testIsEmpty() + { + $metadata = new VolumeMetadata; + $this->assertTrue($metadata->isEmpty()); + $file = new ImageMetadata('filename'); + $metadata->addFile($file); + $this->assertTrue($metadata->isEmpty()); + $file = new ImageMetadata('filename', area: 100); + $metadata->addFile($file); + $this->assertFalse($metadata->isEmpty()); + } + + public function testHasAnnotations() + { + $metadata = new VolumeMetadata; + $this->assertFalse($metadata->hasAnnotations()); + $file = new ImageMetadata('filename'); + $metadata->addFile($file); + $this->assertFalse($metadata->hasAnnotations()); + + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $la = new LabelAndUser($label, $user); + $annotation = new ImageAnnotation( + shape: Shape::point(), + points: [10, 10], + labels: [$la], + ); + $file->addAnnotation($annotation); + $this->assertTrue($metadata->hasAnnotations()); + } + + public function testHasFileLabels() + { + $metadata = new VolumeMetadata; + $this->assertFalse($metadata->hasFileLabels()); + $file = new ImageMetadata('filename'); + $metadata->addFile($file); + $this->assertFalse($metadata->hasFileLabels()); + + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $la = new LabelAndUser($label, $user); + $file->addFileLabel($la); + $this->assertTrue($metadata->hasFileLabels()); + } + + public function testGetAnnotationLabels() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('filename'); + $metadata->addFile($file); + $this->assertEquals([], $metadata->getAnnotationLabels()); + + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $la = new LabelAndUser($label, $user); + $annotation = new ImageAnnotation( + shape: Shape::point(), + points: [10, 10], + labels: [$la], + ); + $file->addAnnotation($annotation); + $this->assertEquals([123 => $label], $metadata->getAnnotationLabels()); + } + + public function testGetAnnotationLabelsOnlyLabels() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('filename'); + $metadata->addFile($file); + + $label1 = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $la = new LabelAndUser($label1, $user); + $annotation = new ImageAnnotation( + shape: Shape::point(), + points: [10, 10], + labels: [$la], + ); + $file->addAnnotation($annotation); + + $label2 = new Label(456, 'my label'); + $la = new LabelAndUser($label2, $user); + $annotation = new ImageAnnotation( + shape: Shape::point(), + points: [10, 10], + labels: [$la], + ); + $file->addAnnotation($annotation); + + $this->assertEquals([123 => $label1], $metadata->getAnnotationLabels([123])); + } + + public function testGetFileLabels() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('filename'); + $metadata->addFile($file); + $this->assertEquals([], $metadata->getFileLabels()); + + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $la = new LabelAndUser($label, $user); + $file->addFileLabel($la); + $this->assertEquals([123 => $label], $metadata->getFileLabels()); + } + + public function testGetFileLabelsOnlyLabels() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('filename'); + $metadata->addFile($file); + + $label1 = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $la = new LabelAndUser($label1, $user); + $file->addFileLabel($la); + + $label2 = new Label(456, 'my label'); + $la = new LabelAndUser($label2, $user); + $file->addFileLabel($la); + + $this->assertEquals([123 => $label1], $metadata->getFileLabels([123])); + } + + public function testGetUsers() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('filename'); + $metadata->addFile($file); + $this->assertEquals([], $metadata->getUsers()); + + $label = new Label(123, 'my label'); + $user1 = new User(321, 'joe user'); + $lau = new LabelAndUser($label, $user1); + $annotation = new ImageAnnotation( + shape: Shape::point(), + points: [10, 10], + labels: [$lau], + ); + $file->addAnnotation($annotation); + $this->assertEquals([321 => $user1], $metadata->getUsers()); + + $user2 = new User(432, 'joe user'); + $lau = new LabelAndUser($label, $user2); + $file->addFileLabel($lau); + $this->assertEquals([321 => $user1, 432 => $user2], $metadata->getUsers()); + } + + public function testGetUsersOnlyLabels() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('filename'); + $metadata->addFile($file); + $label1 = new Label(123, 'my label'); + $user1 = new User(321, 'joe user'); + $lau = new LabelAndUser($label1, $user1); + $file->addFileLabel($lau); + $label2 = new Label(456, 'my label'); + $user2 = new User(654, 'joe user'); + $lau = new LabelAndUser($label2, $user2); + $file->addFileLabel($lau); + + $this->assertEquals([321 => $user1], $metadata->getUsers([123])); + } + + public function testGetMatchingUsersByMap() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('filename'); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $la = new LabelAndUser($label, $user); + $file->addFileLabel($la); + + $dbUser = DbUser::factory()->create(); + $matches = $metadata->getMatchingUsers([321 => $dbUser->id]); + $this->assertEquals([321 => $dbUser->id], $matches); + } + + public function testGetMatchingUsersByUuid() + { + $dbUser = DbUser::factory()->create(); + + $metadata = new VolumeMetadata; + $file = new ImageMetadata('filename'); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user', uuid: $dbUser->uuid); + $la = new LabelAndUser($label, $user); + $file->addFileLabel($la); + + $matches = $metadata->getMatchingUsers(); + $this->assertEquals([321 => $dbUser->id], $matches); + } + + public function testGetMatchingUsersNoMatch() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('filename'); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user', uuid: "398f42c6-0f24-38de-a1c6-3c467fcb4250"); + $la = new LabelAndUser($label, $user); + $file->addFileLabel($la); + + $matches = $metadata->getMatchingUsers([321 => -1]); + $this->assertEquals([321 => null], $matches); + } + + public function testGetMatchingUsersOnlyLabels() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('filename'); + $metadata->addFile($file); + $label1 = new Label(123, 'my label'); + $user1 = new User(321, 'joe user'); + $la = new LabelAndUser($label1, $user1); + $file->addFileLabel($la); + $label2 = new Label(456, 'my label'); + $user2 = new User(654, 'joe user'); + $la = new LabelAndUser($label2, $user2); + $file->addFileLabel($la); + + $dbUser = DbUser::factory()->create(); + $matches = $metadata->getMatchingUsers([321 => $dbUser->id], [123]); + $this->assertEquals([321 => $dbUser->id], $matches); + } + + public function testGetMatchingUsersIgnoreNotInMetadata() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('filename'); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $la = new LabelAndUser($label, $user); + $file->addFileLabel($la); + + $dbUser = DbUser::factory()->create(); + $matches = $metadata->getMatchingUsers([321 => $dbUser->id, 432 => $dbUser->id]); + $this->assertEquals([321 => $dbUser->id], $matches); + } + + public function testGetMatchingLabelsByMap() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('filename'); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $la = new LabelAndUser($label, $user); + $file->addFileLabel($la); + + $dbLabel = DbLabel::factory()->create(); + $matches = $metadata->getMatchingLabels([123 => $dbLabel->id]); + $this->assertEquals([123 => $dbLabel->id], $matches); + } + + public function testGetMatchingLabelsByUuid() + { + $dbLabel = DbLabel::factory()->create(); + + $metadata = new VolumeMetadata; + $file = new ImageMetadata('filename'); + $metadata->addFile($file); + $label = new Label(123, 'my label', uuid: $dbLabel->uuid); + $user = new User(321, 'joe user'); + $la = new LabelAndUser($label, $user); + $file->addFileLabel($la); + + $matches = $metadata->getMatchingLabels(); + $this->assertEquals([123 => $dbLabel->id], $matches); + } + + public function testGetMatchingLabelsNoMatch() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('filename'); + $metadata->addFile($file); + $label = new Label(123, 'my label', uuid: "398f42c6-0f24-38de-a1c6-3c467fcb4250"); + $user = new User(321, 'joe user'); + $la = new LabelAndUser($label, $user); + $file->addFileLabel($la); + + $matches = $metadata->getMatchingLabels([123 => -1]); + $this->assertEquals([123 => null], $matches); + } + + public function testGetMatchingLabelsOnlyLabels() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('filename'); + $metadata->addFile($file); + $label1 = new Label(123, 'my label'); + $user1 = new User(321, 'joe user'); + $la = new LabelAndUser($label1, $user1); + $file->addFileLabel($la); + $label2 = new Label(456, 'my label'); + $user2 = new User(654, 'joe user'); + $la = new LabelAndUser($label2, $user2); + $file->addFileLabel($la); + + $dbLabel = DbLabel::factory()->create(); + $matches = $metadata->getMatchingLabels([123 => $dbLabel->id], [123]); + $this->assertEquals([123 => $dbLabel->id], $matches); + } + + public function testGetMatchingLabelsIgnoreNotInMetadata() + { + $metadata = new VolumeMetadata; + $file = new ImageMetadata('filename'); + $metadata->addFile($file); + $label = new Label(123, 'my label'); + $user = new User(321, 'joe user'); + $la = new LabelAndUser($label, $user); + $file->addFileLabel($la); + + $dbLabel = DbLabel::factory()->create(); + $matches = $metadata->getMatchingLabels([123 => $dbLabel->id, 234 => $dbLabel->id]); + $this->assertEquals([123 => $dbLabel->id], $matches); + } +} diff --git a/tests/php/Traits/ParsesMetadataTest.php b/tests/php/Traits/ParsesMetadataTest.php deleted file mode 100644 index ac6b89a08..000000000 --- a/tests/php/Traits/ParsesMetadataTest.php +++ /dev/null @@ -1,494 +0,0 @@ -assertSame($expect, $stub->parseMetadata($input)); - } - - public function testParseMetadataCaseInsensitive() - { - $stub = new ParsesMetadataStub; - $input = "Filename,tAken_at,lnG,Lat,gPs_altitude,diStance_to_ground,areA\nabc.jpg,2016-12-19 12:27:00,52.220,28.123,-1500,10,2.6"; - $expect = [ - ['filename', 'taken_at', 'lng', 'lat', 'gps_altitude', 'distance_to_ground', 'area'], - ['abc.jpg', '2016-12-19 12:27:00', '52.220', '28.123', '-1500', '10', '2.6'], - ]; - $this->assertSame($expect, $stub->parseMetadata($input)); - } - - public function testParseMetadataSynonyms1() - { - $stub = new ParsesMetadataStub; - $input = "file,lon,lat,heading\nabc.jpg,52.220,28.123,180"; - $expect = [ - ['filename', 'lng', 'lat', 'yaw'], - ['abc.jpg', '52.220', '28.123', '180'], - ]; - $this->assertSame($expect, $stub->parseMetadata($input)); - } - - public function testParseMetadataSynonyms2() - { - $stub = new ParsesMetadataStub; - $input = "file,longitude,latitude\nabc.jpg,52.220,28.123"; - $expect = [ - ['filename', 'lng', 'lat'], - ['abc.jpg', '52.220', '28.123'], - ]; - $this->assertSame($expect, $stub->parseMetadata($input)); - } - - public function testParseMetadataSynonyms3() - { - $stub = new ParsesMetadataStub; - $input = "filename,SUB_datetime,SUB_longitude,SUB_latitude,SUB_altitude,SUB_distance,area,SUB_heading\nabc.jpg,2016-12-19 12:27:00,52.220,28.123,-1500,10,2.6,180"; - $expect = [ - ['filename', 'taken_at', 'lng', 'lat', 'gps_altitude', 'distance_to_ground', 'area', 'yaw'], - ['abc.jpg', '2016-12-19 12:27:00', '52.220', '28.123', '-1500', '10', '2.6', '180'], - ]; - $this->assertSame($expect, $stub->parseMetadata($input)); - } - - public function testParseMetadataEmptyCells() - { - $stub = new ParsesMetadataStub; - $input = "filename,taken_at,lng,lat,gps_altitude,distance_to_ground,area,yaw\nabc.jpg,,52.220,28.123,,,,"; - $expect = [ - ['filename', 'taken_at', 'lng', 'lat', 'gps_altitude', 'distance_to_ground', 'area', 'yaw'], - ['abc.jpg', '', '52.220', '28.123', '', '', '', ''], - ]; - $this->assertSame($expect, $stub->parseMetadata($input)); - } - - public function testParseMetadataFile() - { - $stub = new ParsesMetadataStub; - $csv = __DIR__."/../../files/image-metadata.csv"; - $file = new UploadedFile($csv, 'metadata.csv', 'text/csv', null, true); - $expect = [ - ['filename', 'taken_at', 'lng', 'lat', 'gps_altitude', 'distance_to_ground', 'area', 'yaw'], - ['abc.jpg', '2016-12-19 12:27:00', '52.220', '28.123', '-1500', '10', '2.6', '180'], - ]; - $this->assertSame($expect, $stub->parseMetadataFile($file)); - } - - public function testParseMetadataFileBOM() - { - $stub = new ParsesMetadataStub; - $csv = __DIR__."/../../files/image-metadata-with-bom.csv"; - $file = new UploadedFile($csv, 'metadata.csv', 'text/csv', null, true); - $expect = [ - ['filename', 'taken_at', 'lng', 'lat', 'gps_altitude', 'distance_to_ground', 'area', 'yaw'], - ['abc.jpg', '2016-12-19 12:27:00', '52.220', '28.123', '-1500', '10', '2.6', '180'], - ]; - $this->assertSame($expect, $stub->parseMetadataFile($file)); - } - - public function testParseIfdo() - { - $stub = new ParsesMetadataStub; - $input = << 'myvolume', - 'handle' => '20.500.12085/d7546c4b-307f-4d42-8554-33236c577450', - 'uuid' => 'd7546c4b-307f-4d42-8554-33236c577450', - 'url' => '', - 'media_type' => 'image', - 'files' => [ - ['filename'], - ['myimage.jpg'], - ], - ]; - $this->assertSame($expect, $stub->parseIfdo($input)); - } - - public function testParseIfdoHeader() - { - $stub = new ParsesMetadataStub; - $input = << 'myvolume', - 'handle' => '20.500.12085/d7546c4b-307f-4d42-8554-33236c577450', - 'uuid' => 'd7546c4b-307f-4d42-8554-33236c577450', - 'url' => 'https://hdl.handle.net/20.500.12085/d7546c4b-307f-4d42-8554-33236c577450@data', - 'media_type' => 'image', - 'files' => [ - ['filename', 'area', 'distance_to_ground', 'gps_altitude', 'lat', 'lng', 'taken_at', 'yaw'], - ['myimage.jpg', 5.0, 2, -2248.0, 11.8581802, -117.0214864, '2019-04-06 04:29:27.000000', 20], - ], - ]; - $this->assertSame($expect, $stub->parseIfdo($input)); - } - - public function testParseIfdoVideoType() - { - $stub = new ParsesMetadataStub; - $input = << 'myvolume', - 'handle' => '20.500.12085/d7546c4b-307f-4d42-8554-33236c577450', - 'uuid' => 'd7546c4b-307f-4d42-8554-33236c577450', - 'url' => '', - 'media_type' => 'video', - 'files' => [ - ['filename'], - ['myvideo.mp4'], - ], - ]; - $this->assertSame($expect, $stub->parseIfdo($input)); - } - - public function testParseIfdoSlideIsImageType() - { - $stub = new ParsesMetadataStub; - $input = << 'myvolume', - 'handle' => '20.500.12085/d7546c4b-307f-4d42-8554-33236c577450', - 'uuid' => 'd7546c4b-307f-4d42-8554-33236c577450', - 'url' => '', - 'media_type' => 'image', - 'files' => [ - ['filename'], - ['myimage.jpg'], - ], - ]; - $this->assertSame($expect, $stub->parseIfdo($input)); - } - - public function testParseIfdoItems() - { - $stub = new ParsesMetadataStub; - $input = << 'myvolume', - 'handle' => '20.500.12085/d7546c4b-307f-4d42-8554-33236c577450', - 'uuid' => 'd7546c4b-307f-4d42-8554-33236c577450', - 'url' => '', - 'media_type' => 'image', - 'files' => [ - ['filename', 'area', 'distance_to_ground', 'gps_altitude', 'lat', 'lng', 'taken_at', 'yaw'], - ['myimage.jpg', 5.0, 2, -2248.0, 11.8581802, -117.0214864, '2019-04-06 04:29:27.000000', 20], - ], - ]; - $this->assertSame($expect, $stub->parseIfdo($input)); - } - - public function testParseIfdoImageArrayItems() - { - $stub = new ParsesMetadataStub; - $input = << 'myvolume', - 'handle' => '20.500.12085/d7546c4b-307f-4d42-8554-33236c577450', - 'uuid' => 'd7546c4b-307f-4d42-8554-33236c577450', - 'url' => '', - 'media_type' => 'image', - 'files' => [ - ['filename', 'area', 'distance_to_ground', 'gps_altitude', 'lat', 'lng', 'taken_at', 'yaw'], - ['myimage.jpg', 5.0, 2, -2248.0, 11.8581802, -117.0214864, '2019-04-06 04:29:27.000000', 20], - - ], - ]; - $this->assertSame($expect, $stub->parseIfdo($input)); - } - - public function testParseIfdoItemsOverrideHeader() - { - $stub = new ParsesMetadataStub; - $input = << 'myvolume', - 'handle' => '20.500.12085/d7546c4b-307f-4d42-8554-33236c577450', - 'uuid' => 'd7546c4b-307f-4d42-8554-33236c577450', - 'url' => '', - 'media_type' => 'image', - 'files' => [ - ['filename', 'area', 'distance_to_ground', 'gps_altitude', 'lat', 'lng', 'taken_at', 'yaw'], - ['myimage.jpg', 5.1, 3, -2248.0, 11.8581802, -117.0214864, '2019-04-06 05:29:27.000000', 20], - ], - ]; - $this->assertSame($expect, $stub->parseIfdo($input)); - } - - public function testParseIfdoSubItemsOverrideDefaultsAndHeader() - { - $stub = new ParsesMetadataStub; - $input = << 'myvolume', - 'handle' => '20.500.12085/d7546c4b-307f-4d42-8554-33236c577450', - 'uuid' => 'd7546c4b-307f-4d42-8554-33236c577450', - 'url' => '', - 'media_type' => 'video', - 'files' => [ - ['filename', 'area', 'distance_to_ground', 'gps_altitude', 'lat', 'lng', 'taken_at', 'yaw'], - ['myvideo.mp4', 5.1, 3, -2248.0, 11.8581802, -117.0214864, '2019-04-06 05:29:27.000000', 20], - ['myvideo.mp4', 5.1, 4, -2248.0, 11.8581802, -117.0214864, '2019-04-06 05:30:27.000000', 20], - ], - ]; - $this->assertSame($expect, $stub->parseIfdo($input)); - } - - public function testParseIfdoFile() - { - $stub = new ParsesMetadataStub; - $path = __DIR__."/../../files/image-ifdo.yaml"; - $file = new UploadedFile($path, 'ifdo.yaml', 'application/yaml', null, true); - $expect = [ - 'name' => 'SO268 SO268-2_100-1_OFOS SO_CAM-1_Photo_OFOS', - 'handle' => '20.500.12085/d7546c4b-307f-4d42-8554-33236c577450', - 'uuid' => 'd7546c4b-307f-4d42-8554-33236c577450', - 'url' => 'https://hdl.handle.net/20.500.12085/d7546c4b-307f-4d42-8554-33236c577450@data', - 'media_type' => 'image', - 'files' => [ - ['filename', 'area', 'distance_to_ground', 'gps_altitude', 'lat', 'lng', 'taken_at', 'yaw'], - ['SO268-2_100-1_OFOS_SO_CAM-1_20190406_042927.JPG', 5.0, 2, -2248.0, 11.8581802, -117.0214864, '2019-04-06 04:29:27.000000', 20], - ['SO268-2_100-1_OFOS_SO_CAM-1_20190406_052726.JPG', 5.1, 2.1, -4129.6, 11.8582192, -117.0214286, '2019-04-06 05:27:26.000000', 21], - ], - ]; - $this->assertSame($expect, $stub->parseIfdoFile($file)); - } - - public function testParseIfdoNoHeader() - { - $stub = new ParsesMetadataStub; - $input = <<expectException(Exception::class); - $stub->parseIfdo($input); - } - - public function testParseIfdoNoItems() - { - $stub = new ParsesMetadataStub; - $input = << 'myvolume', - 'handle' => '20.500.12085/d7546c4b-307f-4d42-8554-33236c577450', - 'uuid' => 'd7546c4b-307f-4d42-8554-33236c577450', - 'url' => '', - 'media_type' => 'image', - 'files' => [], - ]; - $this->assertSame($expect, $stub->parseIfdo($input)); - } - - public function testParseIfdoNoName() - { - $stub = new ParsesMetadataStub; - $input = <<expectException(Exception::class); - $stub->parseIfdo($input); - } - - public function testParseIfdoNoHandle() - { - $stub = new ParsesMetadataStub; - $input = <<expectException(Exception::class); - $stub->parseIfdo($input); - } - - public function testParseIfdoNoUuid() - { - $stub = new ParsesMetadataStub; - $input = <<expectException(Exception::class); - $stub->parseIfdo($input); - } - - public function testParseIfdoInvalidHandle() - { - $stub = new ParsesMetadataStub; - $input = <<expectException(Exception::class); - $stub->parseIfdo($input); - } - - public function testParseIfdoInvalidDataHandle() - { - $stub = new ParsesMetadataStub; - $input = <<expectException(Exception::class); - $stub->parseIfdo($input); - } - - public function testParseIfdoInvalidYaml() - { - $stub = new ParsesMetadataStub; - $input = <<expectException(Exception::class); - $stub->parseIfdo($input); - } - - public function testParseIfdoNoYamlArray() - { - $stub = new ParsesMetadataStub; - $this->expectException(Exception::class); - $stub->parseIfdo('abc123'); - } -} - -class ParsesMetadataStub -{ - use ParsesMetadata; -} diff --git a/tests/php/VolumeTest.php b/tests/php/VolumeTest.php index 199caaafc..c47682709 100644 --- a/tests/php/VolumeTest.php +++ b/tests/php/VolumeTest.php @@ -7,17 +7,15 @@ use Biigle\Image; use Biigle\MediaType; use Biigle\Role; +use Biigle\Services\MetadataParsing\ImageCsvParser; use Biigle\Volume; use Cache; use Carbon\Carbon; use Event; -use Exception; use Illuminate\Database\QueryException; use Illuminate\Http\UploadedFile; use ModelTestCase; use Storage; -use Symfony\Component\HttpFoundation\StreamedResponse; -use Symfony\Component\HttpKernel\Exception\NotFoundHttpException; class VolumeTest extends ModelTestCase { @@ -553,76 +551,50 @@ public function testCreatingAsyncAttr() $this->assertFalse($this->model->fresh()->creating_async); } - public function testSaveIfdo() + public function testSaveMetadata() { - $disk = Storage::fake('ifdos'); - $csv = __DIR__."/../files/image-ifdo.yaml"; - $file = new UploadedFile($csv, 'ifdo.yaml', 'application/yaml', null, true); + $disk = Storage::fake('metadata'); + $csv = __DIR__."/../files/image-metadata.csv"; + $file = new UploadedFile($csv, 'metadata.csv', 'text/csv', null, true); - $this->assertFalse($this->model->hasIfdo()); - $this->model->saveIfdo($file); + $this->assertFalse($this->model->hasMetadata()); + $this->model->saveMetadata($file); - $disk->assertExists($this->model->id.'.yaml'); - $this->assertTrue($this->model->hasIfdo()); + $disk->assertExists($this->model->id.'.csv'); + $this->assertTrue($this->model->hasMetadata()); + $this->assertEquals($this->model->id.'.csv', $this->model->metadata_file_path); } - public function testHasIfdo() + public function testDeleteMetadataOnDelete() { - $disk = Storage::fake('ifdos'); - $this->assertFalse($this->model->hasIfdo()); - $disk->put($this->model->id.'.yaml', 'abc'); - $this->assertFalse($this->model->hasIfdo()); - Cache::flush(); - $this->assertTrue($this->model->hasIfdo()); - } - - public function testHasIfdoError() - { - Storage::shouldReceive('disk')->andThrow(Exception::class); - $this->assertFalse($this->model->hasIfdo(true)); - - $this->expectException(Exception::class); - $this->model->hasIfdo(); - } - - public function testDeleteIfdo() - { - $disk = Storage::fake('ifdos'); - $disk->put($this->model->id.'.yaml', 'abc'); - $this->assertTrue($this->model->hasIfdo()); - $this->model->deleteIfdo(); - $disk->assertMissing($this->model->id.'.yaml'); - $this->assertFalse($this->model->hasIfdo()); - } - - public function testDeleteIfdoOnDelete() - { - $disk = Storage::fake('ifdos'); - $disk->put($this->model->id.'.yaml', 'abc'); + $disk = Storage::fake('metadata'); + $disk->put($this->model->id.'.csv', 'abc'); + $this->model->metadata_file_path = $this->model->id.'.csv'; + $this->model->save(); $this->model->delete(); - $disk->assertMissing($this->model->id.'.yaml'); + $disk->assertMissing($this->model->id.'.csv'); } - public function testDownloadIfdoNotFound() + public function testDeleteMetadata() { - $this->expectException(NotFoundHttpException::class); - $this->model->downloadIfdo(); - } - - public function testDownloadIfdo() - { - $disk = Storage::fake('ifdos'); - $disk->put($this->model->id.'.yaml', 'abc'); - $response = $this->model->downloadIfdo(); - $this->assertInstanceOf(StreamedResponse::class, $response); + $disk = Storage::fake('metadata'); + $disk->put($this->model->id.'.csv', 'abc'); + $this->model->metadata_file_path = $this->model->id.'.csv'; + $this->model->save(); + $this->model->deleteMetadata(); + $disk->assertMissing($this->model->id.'.csv'); + $this->assertNull($this->model->fresh()->metadata_file_path); } - public function testGetIfdo() + public function testGetMetadata() { - $disk = Storage::fake('ifdos'); - $this->assertNull($this->model->getIfdo()); - $disk->put($this->model->id.'.yaml', 'abc: def'); - $ifdo = $this->model->getIfdo(); - $this->assertSame(['abc' => 'def'], $ifdo); + $this->assertNull($this->model->getMetadata()); + $disk = Storage::fake('metadata'); + $this->model->metadata_file_path = $this->model->id.'.csv'; + $disk->put($this->model->metadata_file_path, "filename,area\n1.jpg,2.5"); + $this->model->metadata_parser = ImageCsvParser::class; + $metadata = $this->model->getMetadata(); + $fileMeta = $metadata->getFile('1.jpg'); + $this->assertSame(2.5, $fileMeta->area); } }