From 8eb9a155b1ac01fb4a12e4840d7d684ff74af8ec Mon Sep 17 00:00:00 2001 From: dantb Date: Tue, 28 Nov 2023 16:29:25 +0100 Subject: [PATCH 01/18] Add support for bulk copying files to another project --- .../delta/plugins/storage/files/Files.scala | 92 +++++++++++++--- .../files/model/CopyFileDestination.scala | 27 +++++ .../storage/files/model/FileAttributes.scala | 2 +- .../plugins/storage/files/model/FileId.scala | 1 + .../storage/files/model/FileRejection.scala | 20 ++++ .../files/routes/CopyFilePayload.scala | 36 ++++++ .../storage/files/routes/FilesRoutes.scala | 103 ++++++++++++------ .../storage/storages/model/Storage.scala | 8 +- .../storages/model/StorageRejection.scala | 5 +- .../storages/operations/CopyFile.scala | 25 +++++ .../operations/StorageFileRejection.scala | 16 +++ .../operations/disk/DiskStorageCopyFile.scala | 32 ++++++ .../remote/RemoteDiskStorageCopyFile.scala | 33 ++++++ .../client/RemoteDiskStorageClient.scala | 34 ++++++ .../errors/tag-and-rev-copy-error.json | 5 + .../plugins/storage/files/FileFixtures.scala | 62 ++++++----- .../plugins/storage/files/FilesSpec.scala | 64 ++++++++++- .../files/routes/FilesRoutesSpec.scala | 78 ++++++++++++- .../client/RemoteStorageClientSpec.scala | 4 +- .../docs/delta/api/assets/files/copy-put.json | 34 ++++++ .../docs/delta/api/assets/files/copy-put.sh | 10 ++ .../main/paradox/docs/delta/api/files-api.md | 58 +++++++++- .../bluebrain/nexus/tests/HttpClient.scala | 40 +++++-- .../nexus/tests/kg/CopyFileSpec.scala | 52 +++++++++ .../nexus/tests/kg/DiskStorageSpec.scala | 4 +- .../nexus/tests/kg/RemoteStorageSpec.scala | 4 +- .../nexus/tests/kg/S3StorageSpec.scala | 2 +- .../nexus/tests/kg/StorageSpec.scala | 12 +- 28 files changed, 745 insertions(+), 118 deletions(-) create mode 100644 delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/CopyFileDestination.scala create mode 100644 delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/CopyFilePayload.scala create mode 100644 delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/CopyFile.scala create mode 100644 delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageCopyFile.scala create mode 100644 delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageCopyFile.scala create mode 100644 delta/plugins/storage/src/test/resources/errors/tag-and-rev-copy-error.json create mode 100644 docs/src/main/paradox/docs/delta/api/assets/files/copy-put.json create mode 100644 docs/src/main/paradox/docs/delta/api/assets/files/copy-put.sh create mode 100644 tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/CopyFileSpec.scala diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala index fb575da2b3..3f0dcc81ca 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala @@ -3,7 +3,7 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.files import akka.actor.typed.ActorSystem import akka.actor.{ActorSystem => ClassicActorSystem} import akka.http.scaladsl.model.ContentTypes.`application/octet-stream` -import akka.http.scaladsl.model.{ContentType, HttpEntity, Uri} +import akka.http.scaladsl.model.{BodyPartEntity, ContentType, HttpEntity, Uri} import cats.effect.{Clock, IO} import cats.syntax.all._ import ch.epfl.bluebrain.nexus.delta.kernel.cache.LocalCache @@ -19,9 +19,9 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.schemas.{files => fileSchema} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StoragesConfig.{RemoteDiskStorageConfig, StorageTypeConfig} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageRejection.{StorageFetchRejection, StorageIsDeprecated} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageRejection.{DifferentStorageType, InvalidStorageType, StorageFetchRejection, StorageIsDeprecated} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{DigestAlgorithm, Storage, StorageRejection, StorageType} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.{FetchAttributeRejection, FetchFileRejection, SaveFileRejection} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.{CopyFileRejection, FetchAttributeRejection, FetchFileRejection, SaveFileRejection} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.RemoteDiskStorageClient import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{Storages, StoragesStatistics} @@ -195,6 +195,57 @@ final class Files( } yield res }.span("createLink") + /** + * Create a file from a source file potentially in a different organization + * @param sourceId + * File lookup id for the source file + * @param dest + * Project, storage and file details for the file we're creating + */ + def copyTo( + sourceId: FileId, + dest: CopyFileDestination + )(implicit c: Caller): IO[FileResource] = { + for { + file <- fetchSourceFile(sourceId) + (pc, destStorageRef, destStorage) <- fetchDestinationStorage(dest) + _ <- validateStorageTypeForCopy(file.storageType, destStorage) + space <- fetchStorageAvailableSpace(destStorage) + _ <- IO.raiseUnless(space.exists(_ < file.attributes.bytes))( + FileTooLarge(destStorage.storageValue.maxFileSize, space) + ) + iri <- dest.fileId.fold(generateId(pc))(FileId(_, dest.project).expandIri(fetchContext.onCreate).map(_._1)) + destinationDesc <- FileDescription(dest.filename.getOrElse(file.attributes.filename), file.attributes.mediaType) + attributes <- CopyFile(destStorage, remoteDiskStorageClient).apply(file.attributes, destinationDesc).adaptError { + case r: CopyFileRejection => CopyRejection(file.id, file.storage.iri, destStorage.id, r) + } + res <- eval(CreateFile(iri, dest.project, destStorageRef, destStorage.tpe, attributes, c.subject, dest.tag)) + } yield res + }.span("copyFile") + + private def fetchSourceFile(id: FileId)(implicit c: Caller) = + for { + file <- fetch(id) + sourceStorage <- storages.fetch(file.value.storage, id.project) + _ <- validateAuth(id.project, sourceStorage.value.storageValue.readPermission) + } yield file.value + + private def fetchDestinationStorage(dest: CopyFileDestination)(implicit c: Caller) = + for { + pc <- fetchContext.onCreate(dest.project) + (destStorageRef, destStorage) <- fetchActiveStorage(dest.storage, dest.project, pc) + } yield (pc, destStorageRef, destStorage) + + private def validateStorageTypeForCopy(source: StorageType, destination: Storage): IO[Unit] = + IO.raiseWhen(source == StorageType.S3Storage)( + WrappedStorageRejection( + InvalidStorageType(destination.id, source, Set(StorageType.DiskStorage, StorageType.RemoteDiskStorage)) + ) + ) >> + IO.raiseUnless(source == destination.tpe)( + WrappedStorageRejection(DifferentStorageType(destination.id, found = destination.tpe, expected = source)) + ) + /** * Update an existing file * @@ -456,20 +507,31 @@ final class Files( private def extractFileAttributes(iri: Iri, entity: HttpEntity, storage: Storage): IO[FileAttributes] = for { - storageAvailableSpace <- storage.storageValue.capacity.fold(IO.none[Long]) { capacity => - storagesStatistics - .get(storage.id, storage.project) - .redeem( - _ => Some(capacity), - stat => Some(capacity - stat.spaceUsed) - ) - } - (description, source) <- formDataExtractor(iri, entity, storage.storageValue.maxFileSize, storageAvailableSpace) - attributes <- SaveFile(storage, remoteDiskStorageClient, config) - .apply(description, source) - .adaptError { case e: SaveFileRejection => SaveRejection(iri, storage.id, e) } + (description, source) <- extractFormData(iri, storage, entity) + attributes <- saveFile(iri, storage, description, source) } yield attributes + private def extractFormData(iri: Iri, storage: Storage, entity: HttpEntity): IO[(FileDescription, BodyPartEntity)] = + for { + storageAvailableSpace <- fetchStorageAvailableSpace(storage) + (description, source) <- formDataExtractor(iri, entity, storage.storageValue.maxFileSize, storageAvailableSpace) + } yield (description, source) + + private def saveFile(iri: Iri, storage: Storage, description: FileDescription, source: BodyPartEntity) = + SaveFile(storage, remoteDiskStorageClient, config) + .apply(description, source) + .adaptError { case e: SaveFileRejection => SaveRejection(iri, storage.id, e) } + + private def fetchStorageAvailableSpace(storage: Storage): IO[Option[Long]] = + storage.storageValue.capacity.fold(IO.none[Long]) { capacity => + storagesStatistics + .get(storage.id, storage.project) + .redeem( + _ => Some(capacity), + stat => Some(capacity - stat.spaceUsed) + ) + } + private def expandStorageIri(segment: IdSegment, pc: ProjectContext): IO[Iri] = Storages.expandIri(segment, pc).adaptError { case s: StorageRejection => WrappedStorageRejection(s) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/CopyFileDestination.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/CopyFileDestination.scala new file mode 100644 index 0000000000..8cce82a5b6 --- /dev/null +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/CopyFileDestination.scala @@ -0,0 +1,27 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model + +import ch.epfl.bluebrain.nexus.delta.sdk.model.IdSegment +import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef +import ch.epfl.bluebrain.nexus.delta.sourcing.model.Tag.UserTag + +/** + * Details of the file we're creating in the copy + * + * @param project + * Orgnization and project for the new file + * @param fileId + * Optional identifier for the new file + * @param storage + * Optional storage for the new file which must have the same type as the source file's storage + * @param tag + * Optional tag to create the new file with + * @param filename + * Optional filename for the new file. If omitted, the source filename will be used + */ +final case class CopyFileDestination( + project: ProjectRef, + fileId: Option[IdSegment], + storage: Option[IdSegment], + tag: Option[UserTag], + filename: Option[String] +) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileAttributes.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileAttributes.scala index 426d9334e2..65c61683a4 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileAttributes.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileAttributes.scala @@ -28,7 +28,7 @@ import scala.annotation.nowarn * @param mediaType * the optional media type of the file * @param bytes - * the size of the file file in bytes + * the size of the file in bytes * @param digest * the digest information of the file * @param origin diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileId.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileId.scala index 0af0c34bd0..a55f658f1f 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileId.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileId.scala @@ -19,6 +19,7 @@ object FileId { def apply(ref: ResourceRef, project: ProjectRef): FileId = FileId(IdSegmentRef(ref), project) def apply(id: IdSegment, tag: UserTag, project: ProjectRef): FileId = FileId(IdSegmentRef(id, tag), project) def apply(id: IdSegment, rev: Int, project: ProjectRef): FileId = FileId(IdSegmentRef(id, rev), project) + def apply(id: IdSegment, project: ProjectRef): FileId = FileId(IdSegmentRef(id), project) val iriExpander: ExpandIri[InvalidFileId] = new ExpandIri(InvalidFileId.apply) } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileRejection.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileRejection.scala index 2c6a60f187..b54d2efa1f 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileRejection.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileRejection.scala @@ -16,6 +16,7 @@ import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.JsonLdContext.keywords import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.encoder.JsonLdEncoder import ch.epfl.bluebrain.nexus.delta.sdk.marshalling.HttpResponseFields import ch.epfl.bluebrain.nexus.delta.sdk.marshalling.RdfRejectionHandler.all._ +import ch.epfl.bluebrain.nexus.delta.sdk.model.IdSegment import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContext import ch.epfl.bluebrain.nexus.delta.sdk.syntax.httpResponseFieldsSyntax import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef @@ -147,6 +148,12 @@ object FileRejection { s"Linking a file '$id' cannot be performed without a 'filename' or a 'path' that does not end with a filename." ) + /** + * Rejection returned when attempting to fetch a file and including both the target tag and revision. + */ + final case class InvalidFileLookup(id: IdSegment) + extends FileRejection(s"Only one of 'tag' and 'rev' can be used to lookup file '$id'.") + /** * Rejection returned when attempting to create/update a file with a Multipart/Form-Data payload that does not * contain a ''file'' fieldName @@ -235,6 +242,19 @@ object FileRejection { final case class LinkRejection(id: Iri, storageId: Iri, rejection: StorageFileRejection) extends FileRejection(s"File '$id' could not be linked using storage '$storageId'", Some(rejection.loggedDetails)) + /** + * Rejection returned when interacting with the storage operations bundle to copy a file already in storage + */ + final case class CopyRejection( + sourceId: Iri, + sourceStorageId: Iri, + destStorageId: Iri, + rejection: StorageFileRejection + ) extends FileRejection( + s"File '$sourceId' could not be copied from storage '$sourceStorageId' to storage '$destStorageId'", + Some(rejection.loggedDetails) + ) + /** * Signals a rejection caused when interacting with other APIs when fetching a resource */ diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/CopyFilePayload.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/CopyFilePayload.scala new file mode 100644 index 0000000000..6ef65bed38 --- /dev/null +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/CopyFilePayload.scala @@ -0,0 +1,36 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes + +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileId +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection.InvalidFileLookup +import ch.epfl.bluebrain.nexus.delta.sdk.model.IdSegment +import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef +import ch.epfl.bluebrain.nexus.delta.sourcing.model.Tag.UserTag +import io.circe.Decoder + +final case class CopyFilePayload( + destFilename: Option[String], + sourceProj: ProjectRef, + sourceFile: IdSegment, + sourceTag: Option[UserTag], + sourceRev: Option[Int] +) { + def toSourceFileId: Either[InvalidFileLookup, FileId] = (sourceTag, sourceRev) match { + case (Some(tag), None) => Right(FileId(sourceFile, tag, sourceProj)) + case (None, Some(rev)) => Right(FileId(sourceFile, rev, sourceProj)) + case (None, None) => Right(FileId(sourceFile, sourceProj)) + case (Some(_), Some(_)) => Left(InvalidFileLookup(sourceFile)) + } +} + +object CopyFilePayload { + + implicit val dec: Decoder[CopyFilePayload] = Decoder.instance { cur => + for { + destFilename <- cur.get[Option[String]]("destinationFilename") + sourceProj <- cur.get[ProjectRef]("sourceProjectRef") + sourceFileId <- cur.get[String]("sourceFileId").map(IdSegment(_)) + sourceTag <- cur.get[Option[UserTag]]("sourceTag") + sourceRev <- cur.get[Option[Int]]("sourceRev") + } yield CopyFilePayload(destFilename, sourceProj, sourceFileId, sourceTag, sourceRev) + } +} diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutes.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutes.scala index 7686829d3a..f44a284601 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutes.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutes.scala @@ -5,10 +5,11 @@ import akka.http.scaladsl.model.Uri.Path import akka.http.scaladsl.model.headers.Accept import akka.http.scaladsl.model.{ContentType, MediaRange} import akka.http.scaladsl.server._ +import cats.data.EitherT import cats.effect.IO import cats.syntax.all._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection._ -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{File, FileId, FileRejection} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{CopyFileDestination, File, FileId, FileRejection} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.permissions.{read => Read, write => Write} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.FilesRoutes._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{schemas, FileResource, Files} @@ -27,6 +28,7 @@ import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller import ch.epfl.bluebrain.nexus.delta.sdk.implicits._ import ch.epfl.bluebrain.nexus.delta.sdk.model.routes.Tag import ch.epfl.bluebrain.nexus.delta.sdk.model.{BaseUri, IdSegment} +import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef import ch.epfl.bluebrain.nexus.delta.sourcing.model.Tag.UserTag import io.circe.Decoder import io.circe.generic.extras.Configuration @@ -71,9 +73,9 @@ final class FilesRoutes( (baseUriPrefix(baseUri.prefix) & replaceUri("files", schemas.files)) { pathPrefix("files") { extractCaller { implicit caller => - resolveProjectRef.apply { ref => + resolveProjectRef.apply { projectRef => implicit class IndexOps(io: IO[FileResource]) { - def index(m: IndexingMode): IO[FileResource] = io.flatTap(self.index(ref, _, m)) + def index(m: IndexingMode): IO[FileResource] = io.flatTap(self.index(projectRef, _, m)) } concat( @@ -87,30 +89,68 @@ final class FilesRoutes( emit( Created, files - .createLink(storage, ref, filename, mediaType, path, tag) + .createLink(storage, projectRef, filename, mediaType, path, tag) .index(mode) .attemptNarrow[FileRejection] ) }, + // Create a file by copying from another project, without id segment + entity(as[CopyFilePayload]) { c: CopyFilePayload => + val copyTo = CopyFileDestination(projectRef, None, storage, tag, c.destFilename) + + emit(Created, copyFile(projectRef, mode, c, copyTo)) + }, // Create a file without id segment extractRequestEntity { entity => emit( Created, - files.create(storage, ref, entity, tag).index(mode).attemptNarrow[FileRejection] + files.create(storage, projectRef, entity, tag).index(mode).attemptNarrow[FileRejection] ) } ) } }, (idSegment & indexingMode) { (id, mode) => - val fileId = FileId(id, ref) + val fileId = FileId(id, projectRef) concat( pathEndOrSingleSlash { operationName(s"$prefixSegment/files/{org}/{project}/{id}") { concat( (put & pathEndOrSingleSlash) { - parameters("rev".as[Int].?, "storage".as[IdSegment].?, "tag".as[UserTag].?) { - case (None, storage, tag) => + concat( + // Create a file by copying from another project + parameters("storage".as[IdSegment].?, "tag".as[UserTag].?) { case (destStorage, destTag) => + entity(as[CopyFilePayload]) { c: CopyFilePayload => + val copyTo = + CopyFileDestination(projectRef, Some(id), destStorage, destTag, c.destFilename) + + emit(Created, copyFile(projectRef, mode, c, copyTo)) + } + }, + parameters("rev".as[Int], "storage".as[IdSegment].?, "tag".as[UserTag].?) { + case (rev, storage, tag) => + concat( + // Update a Link + entity(as[LinkFile]) { case LinkFile(filename, mediaType, path) => + emit( + files + .updateLink(fileId, storage, filename, mediaType, path, rev, tag) + .index(mode) + .attemptNarrow[FileRejection] + ) + }, + // Update a file + extractRequestEntity { entity => + emit( + files + .update(fileId, storage, rev, entity, tag) + .index(mode) + .attemptNarrow[FileRejection] + ) + } + ) + }, + parameters("storage".as[IdSegment].?, "tag".as[UserTag].?) { case (storage, tag) => concat( // Link a file with id segment entity(as[LinkFile]) { case LinkFile(filename, mediaType, path) => @@ -126,36 +166,19 @@ final class FilesRoutes( extractRequestEntity { entity => emit( Created, - files.create(fileId, storage, entity, tag).index(mode).attemptNarrow[FileRejection] - ) - } - ) - case (Some(rev), storage, tag) => - concat( - // Update a Link - entity(as[LinkFile]) { case LinkFile(filename, mediaType, path) => - emit( - files - .updateLink(fileId, storage, filename, mediaType, path, rev, tag) - .index(mode) - .attemptNarrow[FileRejection] - ) - }, - // Update a file - extractRequestEntity { entity => - emit( files - .update(fileId, storage, rev, entity, tag) + .create(fileId, storage, entity, tag) .index(mode) .attemptNarrow[FileRejection] ) } ) - } + } + ) }, // Deprecate a file (delete & parameter("rev".as[Int])) { rev => - authorizeFor(ref, Write).apply { + authorizeFor(projectRef, Write).apply { emit( files .deprecate(fileId, rev) @@ -168,7 +191,7 @@ final class FilesRoutes( // Fetch a file (get & idSegmentRef(id)) { id => - emitOrFusionRedirect(ref, id, fetch(FileId(id, ref))) + emitOrFusionRedirect(projectRef, id, fetch(FileId(id, projectRef))) } ) } @@ -177,9 +200,9 @@ final class FilesRoutes( operationName(s"$prefixSegment/files/{org}/{project}/{id}/tags") { concat( // Fetch a file tags - (get & idSegmentRef(id) & pathEndOrSingleSlash & authorizeFor(ref, Read)) { id => + (get & idSegmentRef(id) & pathEndOrSingleSlash & authorizeFor(projectRef, Read)) { id => emit( - fetchMetadata(FileId(id, ref)) + fetchMetadata(FileId(id, projectRef)) .map(_.value.tags) .attemptNarrow[FileRejection] .rejectOn[FileNotFound] @@ -187,7 +210,7 @@ final class FilesRoutes( }, // Tag a file (post & parameter("rev".as[Int]) & pathEndOrSingleSlash) { rev => - authorizeFor(ref, Write).apply { + authorizeFor(projectRef, Write).apply { entity(as[Tag]) { case Tag(tagRev, tag) => emit( Created, @@ -198,7 +221,7 @@ final class FilesRoutes( }, // Delete a tag (tagLabel & delete & parameter("rev".as[Int]) & pathEndOrSingleSlash & authorizeFor( - ref, + projectRef, Write )) { (tag, rev) => emit( @@ -213,7 +236,7 @@ final class FilesRoutes( } }, (pathPrefix("undeprecate") & put & parameter("rev".as[Int])) { rev => - authorizeFor(ref, Write).apply { + authorizeFor(projectRef, Write).apply { emit( files .undeprecate(fileId, rev) @@ -231,6 +254,16 @@ final class FilesRoutes( } } + private def copyFile(projectRef: ProjectRef, mode: IndexingMode, c: CopyFilePayload, copyTo: CopyFileDestination)( + implicit caller: Caller + ): IO[Either[FileRejection, FileResource]] = + (for { + _ <- EitherT.right(aclCheck.authorizeForOr(c.sourceProj, Read)(AuthorizationFailed(c.sourceProj.project, Read))) + sourceFileId <- EitherT.fromEither[IO](c.toSourceFileId) + result <- EitherT(files.copyTo(sourceFileId, copyTo).attemptNarrow[FileRejection]) + _ <- EitherT.right[FileRejection](index(projectRef, result, mode)) + } yield result).value + def fetch(id: FileId)(implicit caller: Caller): Route = (headerValueByType(Accept) & varyAcceptHeaders) { case accept if accept.mediaRanges.exists(metadataMediaRanges.contains) => diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/Storage.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/Storage.scala index 35a256462b..d22d6f816e 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/Storage.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/Storage.scala @@ -4,11 +4,11 @@ import akka.actor.ActorSystem import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StoragesConfig.StorageTypeConfig import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.Metadata import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageValue.{DiskStorageValue, RemoteDiskStorageValue, S3StorageValue} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.{DiskStorageFetchFile, DiskStorageSaveFile} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.{DiskStorageCopyFile, DiskStorageFetchFile, DiskStorageSaveFile} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.RemoteDiskStorageClient import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.s3.{S3StorageFetchFile, S3StorageLinkFile, S3StorageSaveFile} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.{FetchAttributes, FetchFile, LinkFile, SaveFile} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{contexts, Storages} import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.ContextValue @@ -89,6 +89,7 @@ object Storage { def saveFile(implicit as: ActorSystem): SaveFile = new DiskStorageSaveFile(this) + def copyFile: CopyFile = new DiskStorageCopyFile(this) } /** @@ -138,6 +139,9 @@ object Storage { def linkFile(client: RemoteDiskStorageClient): LinkFile = new RemoteDiskStorageLinkFile(this, client) + def copyFile(client: RemoteDiskStorageClient): CopyFile = + new RemoteDiskStorageCopyFile(this, client) + def fetchComputedAttributes(client: RemoteDiskStorageClient): FetchAttributes = new RemoteStorageFetchAttributes(value, client) } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/StorageRejection.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/StorageRejection.scala index 0858b82946..a253b45156 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/StorageRejection.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/StorageRejection.scala @@ -143,10 +143,11 @@ object StorageRejection { extends StorageRejection(s"Storage ${id.fold("")(id => s"'$id'")} has invalid JSON-LD payload.") /** - * Rejection returned when attempting to create a storage with an id that already exists. + * Signals an attempt to update/create a storage based on a previous revision with a different storage type * * @param id - * the storage identifier + * @param found + * @param expected */ final case class DifferentStorageType(id: Iri, found: StorageType, expected: StorageType) extends StorageRejection(s"Storage '$id' is of type '$found' and can't be updated to be a '$expected' .") diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/CopyFile.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/CopyFile.scala new file mode 100644 index 0000000000..c23c4976c7 --- /dev/null +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/CopyFile.scala @@ -0,0 +1,25 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations + +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes, FileDescription} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{Storage, StorageType} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.CopyFileRejection +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.RemoteDiskStorageClient + +trait CopyFile { + def apply(source: FileAttributes, dest: FileDescription): IO[FileAttributes] +} + +object CopyFile { + + def apply(storage: Storage, client: RemoteDiskStorageClient): CopyFile = + storage match { + case storage: Storage.DiskStorage => storage.copyFile + case storage: Storage.S3Storage => unsupported(storage.tpe) + case storage: Storage.RemoteDiskStorage => storage.copyFile(client) + } + + private def unsupported(storageType: StorageType): CopyFile = + (_, _) => IO.raiseError(CopyFileRejection.UnsupportedOperation(storageType)) + +} diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/StorageFileRejection.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/StorageFileRejection.scala index b8f6122b90..a3511184cc 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/StorageFileRejection.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/StorageFileRejection.scala @@ -65,6 +65,22 @@ object StorageFileRejection { extends FetchAttributeRejection(rejection.loggedDetails) } + /** + * Rejection returned when a storage cannot fetch a file's attributes + */ + sealed abstract class CopyFileRejection(loggedDetails: String) extends StorageFileRejection(loggedDetails) + + object CopyFileRejection { + + /** + * Rejection performing this operation because the storage does not support it + */ + final case class UnsupportedOperation(tpe: StorageType) + extends FetchAttributeRejection( + s"Copying a file attributes is not supported for storages of type '${tpe.iri}'" + ) + } + /** * Rejection returned when a storage cannot save a file */ diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageCopyFile.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageCopyFile.scala new file mode 100644 index 0000000000..fbe4ffe1dc --- /dev/null +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageCopyFile.scala @@ -0,0 +1,32 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk + +import akka.http.scaladsl.model.Uri +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes, FileDescription} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.DiskStorage +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.CopyFile +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.DiskStorageSaveFile.initLocation + +import java.net.URI +import java.nio.file.{Paths, StandardCopyOption} +import scala.annotation.nowarn + +class DiskStorageCopyFile(storage: DiskStorage) extends CopyFile { + @nowarn + override def apply(source: FileAttributes, dest: FileDescription): IO[FileAttributes] = { + val sourcePath = Paths.get(URI.create(s"file://${source.location.path}")) + for { + (destPath, destRelativePath) <- initLocation(storage.project, storage.value, dest.uuid, dest.filename) + _ <- fs2.io.file.copy[IO](sourcePath, destPath, Seq(StandardCopyOption.COPY_ATTRIBUTES)) + } yield FileAttributes( + uuid = dest.uuid, + location = Uri(destPath.toUri.toString), + path = Uri.Path(destRelativePath.toString), + filename = dest.filename, + mediaType = source.mediaType, + bytes = source.bytes, + digest = source.digest, + origin = source.origin + ) + } +} diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageCopyFile.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageCopyFile.scala new file mode 100644 index 0000000000..71236dbae9 --- /dev/null +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageCopyFile.scala @@ -0,0 +1,33 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote + +import akka.http.scaladsl.model.Uri +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes, FileDescription} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.RemoteDiskStorage +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.CopyFile +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.SaveFile.intermediateFolders +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.RemoteDiskStorageClient + +class RemoteDiskStorageCopyFile( + storage: RemoteDiskStorage, + client: RemoteDiskStorageClient +) extends CopyFile { + + def apply(source: FileAttributes, description: FileDescription): IO[FileAttributes] = { + val destinationPath = Uri.Path(intermediateFolders(storage.project, description.uuid, description.filename)) + client.copyFile(storage.value.folder, source.location.path, destinationPath)(storage.value.endpoint).as { + FileAttributes( + uuid = description.uuid, + location = source.location, // TODO what's the destination absolute path? + path = destinationPath, + filename = description.filename, + mediaType = description.mediaType, + bytes = source.bytes, + digest = source.digest, + origin = source.origin + ) + } + + } + +} diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/RemoteDiskStorageClient.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/RemoteDiskStorageClient.scala index a14bb20a2c..cf2336fbc7 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/RemoteDiskStorageClient.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/RemoteDiskStorageClient.scala @@ -175,6 +175,40 @@ final class RemoteDiskStorageClient(client: HttpClient, getAuthToken: AuthTokenP } } + /** + * Moves a path from the provided ''sourceRelativePath'' to ''destRelativePath'' inside the nexus folder. + * + * @param bucket + * the storage bucket name + * @param sourceRelativePath + * the source relative path location + * @param destRelativePath + * the destination relative path location inside the nexus folder + */ + def copyFile( + bucket: Label, + sourceRelativePath: Path, + destRelativePath: Path + )(implicit baseUri: BaseUri): IO[Unit] = { + getAuthToken(credentials).flatMap { authToken => + val endpoint = baseUri.endpoint / "buckets" / bucket.value / "files" / destRelativePath + val payload = Json.obj("source" -> sourceRelativePath.toString.asJson) + client + .discardBytes(Post(endpoint, payload).withCredentials(authToken), ()) + .adaptError { + // TODO update error + case error @ HttpClientStatusError(_, `NotFound`, _) if !bucketNotFoundType(error) => + MoveFileRejection.FileNotFound(sourceRelativePath.toString) + case error @ HttpClientStatusError(_, `BadRequest`, _) if pathContainsLinksType(error) => + MoveFileRejection.PathContainsLinks(destRelativePath.toString) + case HttpClientStatusError(_, `Conflict`, _) => + MoveFileRejection.ResourceAlreadyExists(destRelativePath.toString) + case error: HttpClientError => + UnexpectedMoveError(sourceRelativePath.toString, destRelativePath.toString, error.asString) + } + } + } + private def bucketNotFoundType(error: HttpClientError): Boolean = error.jsonBody.fold(false)(_.hcursor.get[String](keywords.tpe).toOption.contains("BucketNotFound")) diff --git a/delta/plugins/storage/src/test/resources/errors/tag-and-rev-copy-error.json b/delta/plugins/storage/src/test/resources/errors/tag-and-rev-copy-error.json new file mode 100644 index 0000000000..111259dff1 --- /dev/null +++ b/delta/plugins/storage/src/test/resources/errors/tag-and-rev-copy-error.json @@ -0,0 +1,5 @@ +{ + "@context" : "https://bluebrain.github.io/nexus/contexts/error.json", + "@type" : "InvalidFileLookup", + "reason" : "Only one of 'tag' and 'rev' can be used to lookup file '{{fileId}}'." +} diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FileFixtures.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FileFixtures.scala index 729dd5e4f8..219d674c4d 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FileFixtures.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FileFixtures.scala @@ -2,6 +2,7 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.files import akka.http.scaladsl.model.ContentTypes.`text/plain(UTF-8)` import akka.http.scaladsl.model.{HttpEntity, MessageEntity, Multipart, Uri} +import cats.effect.unsafe.implicits.global import cats.effect.{IO, Ref} import ch.epfl.bluebrain.nexus.delta.kernel.utils.{UUIDF, UrlUtils} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest.ComputedDigest @@ -11,37 +12,39 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{AbsolutePat import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.nxv import ch.epfl.bluebrain.nexus.delta.sdk.generators.ProjectGen import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.ApiMappings -import ch.epfl.bluebrain.nexus.delta.sourcing.model.Label +import ch.epfl.bluebrain.nexus.delta.sourcing.model.{Label, ProjectRef} import ch.epfl.bluebrain.nexus.testkit.scalatest.EitherValues -import ch.epfl.bluebrain.nexus.testkit.scalatest.ce.CatsIOValues import org.scalatest.Suite import java.nio.file.{Files => JavaFiles} import java.util.{Base64, UUID} -trait FileFixtures extends EitherValues with CatsIOValues { +trait FileFixtures extends EitherValues { self: Suite => - val uuid = UUID.fromString("8249ba90-7cc6-4de5-93a1-802c04200dcc") - val uuid2 = UUID.fromString("12345678-7cc6-4de5-93a1-802c04200dcc") - val ref = Ref.of[IO, UUID](uuid).accepted - implicit val uuidF: UUIDF = UUIDF.fromRef(ref) - val org = Label.unsafe("org") - val orgDeprecated = Label.unsafe("org-deprecated") - val project = ProjectGen.project("org", "proj", base = nxv.base, mappings = ApiMappings("file" -> schemas.files)) - val deprecatedProject = ProjectGen.project("org", "proj-deprecated") - val projectWithDeprecatedOrg = ProjectGen.project("org-deprecated", "other-proj") - val projectRef = project.ref - val diskId2 = nxv + "disk2" - val file1 = nxv + "file1" - val file2 = nxv + "file2" - val fileTagged = nxv + "fileTagged" - val fileTagged2 = nxv + "fileTagged2" - val file1Encoded = UrlUtils.encode(file1.toString) - val encodeId = (id: String) => UrlUtils.encode((nxv + id).toString) - val generatedId = project.base.iri / uuid.toString - val generatedId2 = project.base.iri / uuid2.toString + val uuid = UUID.fromString("8249ba90-7cc6-4de5-93a1-802c04200dcc") + val uuid2 = UUID.fromString("12345678-7cc6-4de5-93a1-802c04200dcc") + val uuidOrg2 = UUID.fromString("66666666-7cc6-4de5-93a1-802c04200dcc") + val ref = Ref.of[IO, UUID](uuid).unsafeRunSync() + implicit val uuidF: UUIDF = UUIDF.fromRef(ref) + val org = Label.unsafe("org") + val org2 = Label.unsafe("org2") + val project = ProjectGen.project(org.value, "proj", base = nxv.base, mappings = ApiMappings("file" -> schemas.files)) + val project2 = + ProjectGen.project(org2.value, "proj2", base = nxv.base, mappings = ApiMappings("file" -> schemas.files)) + val deprecatedProject = ProjectGen.project("org", "proj-deprecated") + val projectRef = project.ref + val projectRefOrg2 = project2.ref + val diskId2 = nxv + "disk2" + val file1 = nxv + "file1" + val file2 = nxv + "file2" + val fileTagged = nxv + "fileTagged" + val fileTagged2 = nxv + "fileTagged2" + val file1Encoded = UrlUtils.encode(file1.toString) + val encodeId = (id: String) => UrlUtils.encode((nxv + id).toString) + val generatedId = project.base.iri / uuid.toString + val generatedId2 = project.base.iri / uuid2.toString val content = "file content" val path = AbsolutePath(JavaFiles.createTempDirectory("files")).rightValue @@ -50,16 +53,21 @@ trait FileFixtures extends EitherValues with CatsIOValues { def withUUIDF[T](id: UUID)(test: => T): T = (for { old <- ref.getAndSet(id) - t <- IO.delay(test).onError(_ => ref.set(old)) + t <- IO(test).onError(_ => ref.set(old)) _ <- ref.set(old) - } yield t).accepted + } yield t).unsafeRunSync() - def attributes(filename: String = "file.txt", size: Long = 12, id: UUID = uuid): FileAttributes = { + def attributes( + filename: String = "file.txt", + size: Long = 12, + id: UUID = uuid, + projRef: ProjectRef = projectRef + ): FileAttributes = { val uuidPathSegment = id.toString.take(8).mkString("/") FileAttributes( id, - s"file://$path/org/proj/$uuidPathSegment/$filename", - Uri.Path(s"org/proj/$uuidPathSegment/$filename"), + s"file://$path/${projRef.toString}/$uuidPathSegment/$filename", + Uri.Path(s"${projRef.toString}/$uuidPathSegment/$filename"), filename, Some(`text/plain(UTF-8)`), size, diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala index 9f6e2aa4bc..6ed0ab682c 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala @@ -11,8 +11,8 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.RemoteContextResolutionFixt import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest.NotComputedDigest import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes.FileAttributesOrigin.Storage import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection._ -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes, FileId, FileRejection} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageRejection.StorageNotFound +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{CopyFileDestination, FileAttributes, FileId, FileRejection} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageRejection.{DifferentStorageType, StorageNotFound} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageType.{RemoteDiskStorage => RemoteStorageType} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{StorageRejection, StorageStatEntry, StorageType} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.AkkaSourceHelpers @@ -26,7 +26,7 @@ import ch.epfl.bluebrain.nexus.delta.sdk.acls.model.AclAddress import ch.epfl.bluebrain.nexus.delta.sdk.auth.{AuthTokenProvider, Credentials} import ch.epfl.bluebrain.nexus.delta.sdk.directives.FileResponse import ch.epfl.bluebrain.nexus.delta.sdk.error.ServiceError.AuthorizationFailed -import ch.epfl.bluebrain.nexus.delta.sdk.http.{HttpClient, HttpClientConfig} +import ch.epfl.bluebrain.nexus.delta.sdk.http.HttpClient import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.{Caller, ServiceAccount} import ch.epfl.bluebrain.nexus.delta.sdk.implicits._ import ch.epfl.bluebrain.nexus.delta.sdk.model._ @@ -62,9 +62,8 @@ class FilesSpec(docker: RemoteStorageDocker) private val alice = User("Alice", realm) "The Files operations bundle" when { - implicit val hcc: HttpClientConfig = httpClientConfig implicit val typedSystem: typed.ActorSystem[Nothing] = system.toTyped - implicit val httpClient: HttpClient = HttpClient() + implicit val httpClient: HttpClient = HttpClient()(httpClientConfig, system) implicit val caller: Caller = Caller(bob, Set(bob, Group("mygroup", realm), Authenticated(realm))) implicit val authTokenProvider: AuthTokenProvider = AuthTokenProvider.anonymousForTest val remoteDiskStorageClient = new RemoteDiskStorageClient(httpClient, authTokenProvider, Credentials.Anonymous) @@ -92,13 +91,14 @@ class FilesSpec(docker: RemoteStorageDocker) val storage: IdSegment = nxv + "other-storage" val fetchContext = FetchContextDummy( - Map(project.ref -> project.context), + Map(project.ref -> project.context, project2.ref -> project2.context), Set(deprecatedProject.ref) ) val aclCheck = AclSimpleCheck( (Anonymous, AclAddress.Root, Set(Permissions.resources.read)), (bob, AclAddress.Project(projectRef), Set(diskFields.readPermission.value, diskFields.writePermission.value)), + (bob, AclAddress.Project(projectRefOrg2), Set(diskFields.readPermission.value, diskFields.writePermission.value)), (alice, AclAddress.Project(projectRef), Set(otherRead, otherWrite)) ).accepted @@ -153,10 +153,12 @@ class FilesSpec(docker: RemoteStorageDocker) "create storages for files" in { val payload = diskFieldsJson deepMerge json"""{"capacity": 320, "maxFileSize": 300, "volume": "$path"}""" storages.create(diskId, projectRef, payload).accepted + storages.create(diskId, projectRefOrg2, payload).accepted val payload2 = json"""{"@type": "RemoteDiskStorage", "endpoint": "${docker.hostConfig.endpoint}", "folder": "${RemoteStorageDocker.BucketName}", "readPermission": "$otherRead", "writePermission": "$otherWrite", "maxFileSize": 300, "default": false}""" storages.create(remoteId, projectRef, payload2).accepted + storages.create(remoteId, projectRefOrg2, payload2).accepted } "succeed with the id passed" in { @@ -438,6 +440,56 @@ class FilesSpec(docker: RemoteStorageDocker) } } + "copying a file" should { + + "succeed from disk storage based on a tag" in { + val newFileId = genString() + val destination = CopyFileDestination(projectRefOrg2, Some(newFileId), None, None, None) + val expectedFilename = "myfile.txt" + val expectedAttr = attributes(filename = expectedFilename, projRef = projectRefOrg2) + val expected = mkResource(nxv + newFileId, projectRefOrg2, diskRev, expectedAttr) + + val actual = files.copyTo(FileId("file1", tag, projectRef), destination).accepted + actual shouldEqual expected + + val fetched = files.fetch(FileId(newFileId, projectRefOrg2)).accepted + fetched shouldEqual expected + } + + "succeed from disk storage based on a rev and should tag the new file" in { + val (newFileId, newTag) = (genString(), UserTag.unsafe(genString())) + val destination = + CopyFileDestination(projectRefOrg2, Some(newFileId), None, Some(newTag), None) + val expectedFilename = "file.txt" + val expectedAttr = attributes(filename = expectedFilename, projRef = projectRefOrg2) + val expected = mkResource(nxv + newFileId, projectRefOrg2, diskRev, expectedAttr, tags = Tags(newTag -> 1)) + + val actual = files.copyTo(FileId("file1", 2, projectRef), destination).accepted + actual shouldEqual expected + + val fetchedByTag = files.fetch(FileId(newFileId, newTag, projectRefOrg2)).accepted + fetchedByTag shouldEqual expected + } + + "reject if the source file doesn't exist" in { + val destination = CopyFileDestination(projectRefOrg2, None, None, None, None) + files.copyTo(fileIdIri(nxv + "other"), destination).rejectedWith[FileNotFound] + } + + "reject if the destination storage doesn't exist" in { + val destination = CopyFileDestination(projectRefOrg2, None, Some(storage), None, None) + files.copyTo(fileId("file1"), destination).rejected shouldEqual + WrappedStorageRejection(StorageNotFound(storageIri, projectRefOrg2)) + } + + "reject if copying between different storage types" in { + val expectedError = DifferentStorageType(remoteIdIri, StorageType.RemoteDiskStorage, StorageType.DiskStorage) + val destination = CopyFileDestination(projectRefOrg2, None, Some(remoteId), None, None) + files.copyTo(FileId("file1", projectRef), destination).rejected shouldEqual + WrappedStorageRejection(expectedError) + } + } + "deleting a tag" should { "succeed" in { val expected = mkResource(file1, projectRef, diskRev, attributes(), rev = 4) diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala index cabb839c63..86a15f625f 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala @@ -43,8 +43,11 @@ import ch.epfl.bluebrain.nexus.testkit.ce.IOFromMap import ch.epfl.bluebrain.nexus.testkit.errors.files.FileErrors.{fileAlreadyExistsError, fileIsNotDeprecatedError} import ch.epfl.bluebrain.nexus.testkit.scalatest.ce.CatsIOValues import io.circe.Json +import io.circe.syntax.KeyOps import org.scalatest._ +import java.util.UUID + class FilesRoutesSpec extends BaseRouteSpec with CancelAfterFailure @@ -88,7 +91,7 @@ class FilesRoutesSpec private val asWriter = addCredentials(OAuth2BearerToken("writer")) private val asS3Writer = addCredentials(OAuth2BearerToken("s3writer")) - private val fetchContext = FetchContextDummy(Map(project.ref -> project.context)) + private val fetchContext = FetchContextDummy(Map(project.ref -> project.context, project2.ref -> project2.context)) private val s3Read = Permission.unsafe("s3/read") private val s3Write = Permission.unsafe("s3/write") @@ -137,8 +140,11 @@ class FilesRoutesSpec clock )(uuidF, typedSystem) private val groupDirectives = - DeltaSchemeDirectives(fetchContext, ioFromMap(uuid -> projectRef.organization), ioFromMap(uuid -> projectRef)) - + DeltaSchemeDirectives( + fetchContext, + ioFromMap(uuid -> projectRef.organization, uuidOrg2 -> projectRefOrg2.organization), + ioFromMap(uuid -> projectRef, uuidOrg2 -> projectRefOrg2) + ) private lazy val routes = routesWithIdentities(identities) private def routesWithIdentities(identities: Identities) = Route.seal(FilesRoutes(stCfg, identities, aclCheck, files, groupDirectives, IndexingAction.noop)) @@ -166,6 +172,12 @@ class FilesRoutesSpec .create(dId, projectRef, diskFieldsJson deepMerge defaults deepMerge json"""{"capacity":5000}""")(callerWriter) .void .accepted + storages + .create(dId, projectRefOrg2, diskFieldsJson deepMerge defaults deepMerge json"""{"capacity":5000}""")( + callerWriter + ) + .void + .accepted } "File routes" should { @@ -352,6 +364,60 @@ class FilesRoutesSpec } } + "copy a file" in { + givenAFileInProject(projectRef.toString) { oldFileId => + val newFileId = genString() + val json = Json.obj("sourceProjectRef" := projectRef, "sourceFileId" := oldFileId) + + Put(s"/v1/files/${projectRefOrg2.toString}/$newFileId", json.toEntity) ~> asWriter ~> routes ~> check { + status shouldEqual StatusCodes.Created + val expectedId = project2.base.iri / newFileId + val attr = attributes(filename = oldFileId) + response.asJson shouldEqual fileMetadata(projectRefOrg2, expectedId, attr, diskIdRev) + } + } + } + + "copy a file with generated new Id" in { + val fileCopyUUId = UUID.randomUUID() + withUUIDF(fileCopyUUId) { + givenAFileInProject(projectRef.toString) { oldFileId => + val json = Json.obj("sourceProjectRef" := projectRef, "sourceFileId" := oldFileId) + + Post(s"/v1/files/${projectRefOrg2.toString}/", json.toEntity) ~> asWriter ~> routes ~> check { + status shouldEqual StatusCodes.Created + val expectedId = project2.base.iri / fileCopyUUId.toString + val attr = attributes(filename = oldFileId, id = fileCopyUUId) + response.asJson shouldEqual fileMetadata(projectRefOrg2, expectedId, attr, diskIdRev) + } + } + } + } + + "reject file copy request if tag and rev are present simultaneously" in { + givenAFileInProject(projectRef.toString) { oldFileId => + val json = Json.obj( + "sourceProjectRef" := projectRef, + "sourceFileId" := oldFileId, + "sourceTag" := "mytag", + "sourceRev" := 3 + ) + + val requests = List( + Put(s"/v1/files/${projectRefOrg2.toString}/${genString()}", json.toEntity), + Post(s"/v1/files/${projectRefOrg2.toString}/", json.toEntity) + ) + + forAll(requests) { req => + req ~> asWriter ~> routes ~> check { + status shouldEqual StatusCodes.BadRequest + response.asJson shouldEqual + jsonContentOf("/errors/tag-and-rev-copy-error.json", "fileId" -> oldFileId) + } + } + } + } + "deprecate a file" in { givenAFile { id => Delete(s"/v1/files/org/proj/$id?rev=1") ~> asWriter ~> routes ~> check { @@ -632,9 +698,11 @@ class FilesRoutesSpec } } - def givenAFile(test: String => Assertion): Assertion = { + def givenAFile(test: String => Assertion): Assertion = givenAFileInProject("org/proj")(test) + + def givenAFileInProject(projRef: String)(test: String => Assertion): Assertion = { val id = genString() - Put(s"/v1/files/org/proj/$id", entity(s"$id")) ~> asWriter ~> routes ~> check { + Put(s"/v1/files/$projRef/$id", entity(s"$id")) ~> asWriter ~> routes ~> check { status shouldEqual StatusCodes.Created } test(id) diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/RemoteStorageClientSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/RemoteStorageClientSpec.scala index 365c8f1011..aba6c62fa8 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/RemoteStorageClientSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/RemoteStorageClientSpec.scala @@ -86,7 +86,9 @@ class RemoteStorageClientSpec(docker: RemoteStorageDocker) } "move a file" in { - client.moveFile(bucket, Uri.Path("my/file-1.txt"), Uri.Path("other/file-1.txt"))(baseUri).accepted shouldEqual + client + .moveFile(bucket, Uri.Path("my/file-1.txt"), Uri.Path("other/file-1.txt"))(baseUri) + .accepted shouldEqual attributes.copy( location = s"file:///app/$BucketName/nexus/other/file-1.txt", digest = NotComputedDigest diff --git a/docs/src/main/paradox/docs/delta/api/assets/files/copy-put.json b/docs/src/main/paradox/docs/delta/api/assets/files/copy-put.json new file mode 100644 index 0000000000..f4f6a287c1 --- /dev/null +++ b/docs/src/main/paradox/docs/delta/api/assets/files/copy-put.json @@ -0,0 +1,34 @@ +{ + "@context": [ + "https://bluebrain.github.io/nexus/contexts/files.json", + "https://bluebrain.github.io/nexus/contexts/metadata.json" + ], + "@id": "http://localhost:8080/v1/resources/myorg/myproject/_/newfileid", + "@type": "File", + "_bytes": 5963969, + "_constrainedBy": "https://bluebrain.github.io/nexus/schemas/files.json", + "_createdAt": "2021-05-12T07:30:54.576Z", + "_createdBy": "http://localhost:8080/v1/anonymous", + "_deprecated": false, + "_digest": { + "_algorithm": "SHA-256", + "_value": "d14a7cb4602a2c6e1e7035809aa319d07a6d3c58303ecce7804d2e481cd4965f" + }, + "_filename": "newfile.pdf", + "_incoming": "http://localhost:8080/v1/files/myorg/myproject/newfileid/incoming", + "_location": "file:///tmp/test/nexus/myorg/myproject/c/b/5/c/4/d/8/e/newfile.pdf", + "_mediaType": "application/pdf", + "_origin": "Client", + "_outgoing": "http://localhost:8080/v1/files/myorg/myproject/newfileid/outgoing", + "_project": "http://localhost:8080/v1/projects/myorg/myproject", + "_rev": 1, + "_self": "http://localhost:8080/v1/files/myorg/myproject/newfileid", + "_storage": { + "@id": "http://localhost:8080/v1/resources/myorg/myproject/_/remote", + "@type": "RemoteDiskStorage", + "_rev": 1 + }, + "_updatedAt": "2021-05-12T07:30:54.576Z", + "_updatedBy": "http://localhost:8080/v1/anonymous", + "_uuid": "cb5c4d8e-0189-49ab-b761-c92b2d4f49d2" +} \ No newline at end of file diff --git a/docs/src/main/paradox/docs/delta/api/assets/files/copy-put.sh b/docs/src/main/paradox/docs/delta/api/assets/files/copy-put.sh new file mode 100644 index 0000000000..1de7f91d50 --- /dev/null +++ b/docs/src/main/paradox/docs/delta/api/assets/files/copy-put.sh @@ -0,0 +1,10 @@ +curl -X PUT \ + -H "Content-Type: application/json" \ + "http://localhost:8080/v1/files/myorg/myproject/newfileid?storage=remote" -d \ + '{ + "destinationFilename": "newfile.pdf", + "sourceProjectRef": "otherorg/otherproj", + "sourceFileId": "oldfileid", + "sourceTag": "mytag", + "sourceRev": null + }' \ No newline at end of file diff --git a/docs/src/main/paradox/docs/delta/api/files-api.md b/docs/src/main/paradox/docs/delta/api/files-api.md index 1d326b3a7b..450877ac2e 100644 --- a/docs/src/main/paradox/docs/delta/api/files-api.md +++ b/docs/src/main/paradox/docs/delta/api/files-api.md @@ -89,6 +89,62 @@ Request Response : @@snip [created-put.json](assets/files/created-put.json) +## Create copy using POST or PUT + +Create a file copy based on a source file potentially in a different organization. No `MIME` details are necessary since this is not a file upload. Metadata such as the size and digest of the source file are preserved. + +The caller must have the following permissions: +- `files/read` on the source project. +- `storages/write` on the storage in the destination project. + +Either `POST` or `PUT` can be used to copy a file, as with other creation operations. These REST resources are in the context of the **destination** file; the one being created. +- `POST` will generate a new UUID for the file: + ``` + POST /v1/files/{org_label}/{project_label}?storage={storageId}&tag={tagName} + ``` +- `PUT` accepts a `{file_id}` from the user: + ``` + PUT /v1/files/{org_label}/{project_label}/{file_id}?storage={storageId}&tag={tagName} + ``` + +... where +- `{storageId}` optionally selects a specific storage backend for the new file. The `@type` of this storage must be `DiskStorage` or `RemoteDiskStorage`. + If omitted, the default storage of the project is used. The request will be rejected if there's not enough space on the storage. +- `{tagName}` an optional label given to the new file on its first revision. + +Both requests accept the following JSON payload: +```json +{ + "destinationFilename": "{destinationFilename}", + "sourceProjectRef": "{sourceOrg}/{sourceProj}", + "sourceFileId": "{sourceFileId}", + "sourceTag": "{sourceTagName}", + "sourceRev": "{sourceRev}" +} +``` + +... where +- `{destinationFilename}` the optional filename for the new file. If omitted, the source filename will be used. +- `{sourceOrg}` the organization label of the source file. +- `{sourceProj}` the project label of the source file. +- `{sourceFileId}` the unique identifier of the source file. +- `{sourceTagName}` the optional source revision to be fetched. +- `{sourceRev}` the optional source tag to be fetched. + +Notes: + +- The storage type of `sourceFileId` must match that of the destination file. For example, if the destination `storageId` is omitted, the source storage must be of type `DiskStorage` (the default storage type). +- `sourceTagName` and `sourceRev` cannot be simultaneously present. If neither are present, the latest revision of the source file will be used. + +**Example** + +Request +: @@snip [copy-put.sh](assets/files/copy-put.sh) + +Response +: @@snip [copy-put.json](assets/files/copy-put.json) + + ## Link using POST Brings a file existing in a storage to Nexus Delta as a file resource. This operation is supported for files using `S3Storage` and `RemoteDiskStorage`. @@ -106,7 +162,7 @@ POST /v1/files/{org_label}/{project_label}?storage={storageId}&tag={tagName} When not specified, the default storage of the project is used. - `{path}`: String - the relative location (from the point of view of storage folder) on the remote storage where the file exists. - `{filename}`: String - the name that will be given to the file during linking. This field is optional. When not specified, the original filename is retained. -- `{mediaType}`: String - the MediaType fo the file. This field is optional. When not specified, Nexus Delta will attempt to detectput +- `{mediaType}`: String - the MediaType fo the file. This field is optional. When not specified, Nexus Delta will attempt to detect it. - `{tagName}` an optional label given to the linked file resource on its first revision. **Example** diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/HttpClient.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/HttpClient.scala index cf1247141f..bfe91b256e 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/HttpClient.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/HttpClient.scala @@ -66,6 +66,11 @@ class HttpClient private (baseUrl: Uri, httpExt: HttpExt)(implicit )(implicit um: FromEntityUnmarshaller[A]): IO[Assertion] = requestAssert(PUT, url, Some(body), identity, extraHeaders)(assertResponse) + def putAndReturn[A](url: String, body: Json, identity: Identity, extraHeaders: Seq[HttpHeader] = jsonHeaders)( + assertResponse: (A, HttpResponse) => (A, Assertion) + )(implicit um: FromEntityUnmarshaller[A]): IO[A] = + requestAssertAndReturn(PUT, url, Some(body), identity, extraHeaders)(assertResponse).map(_._1) + def putIO[A](url: String, body: IO[Json], identity: Identity, extraHeaders: Seq[HttpHeader] = jsonHeaders)( assertResponse: (A, HttpResponse) => Assertion )(implicit um: FromEntityUnmarshaller[A]): IO[Assertion] = { @@ -151,25 +156,25 @@ class HttpClient private (baseUrl: Uri, httpExt: HttpExt)(implicit )(implicit um: FromEntityUnmarshaller[A]): IO[Assertion] = requestAssert(DELETE, url, None, identity, extraHeaders)(assertResponse) - def requestAssert[A]( + def requestAssertAndReturn[A]( method: HttpMethod, url: String, body: Option[Json], identity: Identity, extraHeaders: Seq[HttpHeader] = jsonHeaders - )(assertResponse: (A, HttpResponse) => Assertion)(implicit um: FromEntityUnmarshaller[A]): IO[Assertion] = { + )(assertResponse: (A, HttpResponse) => (A, Assertion))(implicit um: FromEntityUnmarshaller[A]): IO[(A, Assertion)] = { def buildClue(a: A, response: HttpResponse) = s""" - |Endpoint: ${method.value} $url - |Identity: $identity - |Token: ${Option(tokensMap.get(identity)).map(_.credentials.token()).getOrElse("None")} - |Status code: ${response.status} - |Body: ${body.getOrElse("None")} - |Response: - |$a - |""".stripMargin - - requestJson( + |Endpoint: ${method.value} $url + |Identity: $identity + |Token: ${Option(tokensMap.get(identity)).map(_.credentials.token()).getOrElse("None")} + |Status code: ${response.status} + |Body: ${body.getOrElse("None")} + |Response: + |$a + |""".stripMargin + + requestJson[A, (A, Assertion)]( method, url, body, @@ -179,6 +184,17 @@ class HttpClient private (baseUrl: Uri, httpExt: HttpExt)(implicit ) } + def requestAssert[A]( + method: HttpMethod, + url: String, + body: Option[Json], + identity: Identity, + extraHeaders: Seq[HttpHeader] = jsonHeaders + )(assertResponse: (A, HttpResponse) => Assertion)(implicit um: FromEntityUnmarshaller[A]): IO[Assertion] = + requestAssertAndReturn[A](method, url, body, identity, extraHeaders) { (a, resp) => + (a, assertResponse(a, resp)) + }.map(_._2) + def sparqlQuery[A](url: String, query: String, identity: Identity, extraHeaders: Seq[HttpHeader] = Nil)( assertResponse: (A, HttpResponse) => Assertion )(implicit um: FromEntityUnmarshaller[A]): IO[Assertion] = { diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/CopyFileSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/CopyFileSpec.scala new file mode 100644 index 0000000000..19b4c60858 --- /dev/null +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/CopyFileSpec.scala @@ -0,0 +1,52 @@ +package ch.epfl.bluebrain.nexus.tests.kg + +import akka.http.scaladsl.model._ +import akka.util.ByteString +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.kernel.utils.UrlUtils +import ch.epfl.bluebrain.nexus.tests.HttpClient._ +import ch.epfl.bluebrain.nexus.tests.Identity.storages.Coyote +import ch.epfl.bluebrain.nexus.tests.Optics +import io.circe.Json +import io.circe.syntax.KeyOps +import org.scalatest.Assertion + +trait CopyFileSpec { self: StorageSpec => + + "Copying a json file to a different organization" should { + + def givenAProjectWithStorage(test: String => IO[Assertion]): IO[Assertion] = { + val (proj, org) = (genId(), genId()) + val projRef = s"$org/$proj" + createProjects(Coyote, org, proj) >> + createStorages(projRef) >> + test(projRef) + } + + "succeed" in { + givenAProjectWithStorage { destProjRef => + val sourceFileId = "attachment.json" + val destFileId = "attachment2.json" + val destFilename = genId() + + val payload = Json.obj( + "destinationFilename" := destFilename, + "sourceProjectRef" := self.projectRef, + "sourceFileId" := sourceFileId + ) + val uri = s"/files/$destProjRef/$destFileId?storage=nxv:$storageId" + + for { + json <- deltaClient.putAndReturn[Json](uri, payload, Coyote) { (json, response) => + (json, expectCreated(json, response)) + } + returnedId = Optics.`@id`.getOption(json).getOrElse(fail("could not find @id of created resource")) + assertion <- + deltaClient.get[ByteString](s"/files/$destProjRef/${UrlUtils.encode(returnedId)}", Coyote, acceptAll) { + expectDownload(destFilename, ContentTypes.`application/json`, updatedJsonFileContent) + } + } yield assertion + } + } + } +} diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/DiskStorageSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/DiskStorageSpec.scala index f4f3b64d69..1ae4a01753 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/DiskStorageSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/DiskStorageSpec.scala @@ -8,7 +8,7 @@ import ch.epfl.bluebrain.nexus.tests.iam.types.Permission import io.circe.Json import org.scalatest.Assertion -class DiskStorageSpec extends StorageSpec { +class DiskStorageSpec extends StorageSpec with CopyFileSpec { override def storageName: String = "disk" @@ -32,7 +32,7 @@ class DiskStorageSpec extends StorageSpec { ): _* ) - override def createStorages: IO[Assertion] = { + override def createStorages(projectRef: String): IO[Assertion] = { val payload = jsonContentOf("kg/storages/disk.json") val payload2 = jsonContentOf("kg/storages/disk-perms.json") diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/RemoteStorageSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/RemoteStorageSpec.scala index 52abb4c436..ae9a9f0853 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/RemoteStorageSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/RemoteStorageSpec.scala @@ -17,7 +17,7 @@ import org.scalatest.Assertion import scala.annotation.nowarn import scala.sys.process._ -class RemoteStorageSpec extends StorageSpec { +class RemoteStorageSpec extends StorageSpec with CopyFileSpec { override def storageName: String = "external" @@ -60,7 +60,7 @@ class RemoteStorageSpec extends StorageSpec { ): _* ) - override def createStorages: IO[Assertion] = { + override def createStorages(projectRef: String): IO[Assertion] = { val payload = jsonContentOf( "kg/storages/remote-disk.json", "endpoint" -> externalEndpoint, diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/S3StorageSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/S3StorageSpec.scala index 51af5b062c..d461207fa7 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/S3StorageSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/S3StorageSpec.scala @@ -82,7 +82,7 @@ class S3StorageSpec extends StorageSpec { ): _* ) - override def createStorages: IO[Assertion] = { + override def createStorages(projectRef: String): IO[Assertion] = { val payload = jsonContentOf( "kg/storages/s3.json", "storageId" -> s"https://bluebrain.github.io/nexus/vocabulary/$storageId", diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/StorageSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/StorageSpec.scala index 630e4260db..42e3104295 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/StorageSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/StorageSpec.scala @@ -39,7 +39,7 @@ abstract class StorageSpec extends BaseIntegrationSpec { def locationPrefix: Option[String] - def createStorages: IO[Assertion] + def createStorages(projectRef: String): IO[Assertion] protected def fileSelf(project: String, id: String): String = { val uri = Uri(s"${config.deltaUri}/files/$project") @@ -48,6 +48,9 @@ abstract class StorageSpec extends BaseIntegrationSpec { private[tests] val fileSelfPrefix = fileSelf(projectRef, attachmentPrefix) + val jsonFileContent = """{ "initial": ["is", "a", "test", "file"] }""" + val updatedJsonFileContent = """{ "updated": ["is", "a", "test", "file"] }""" + override def beforeAll(): Unit = { super.beforeAll() createProjects(Coyote, orgId, projId).accepted @@ -55,7 +58,7 @@ abstract class StorageSpec extends BaseIntegrationSpec { "Creating a storage" should { s"succeed for a $storageName storage" in { - createStorages + createStorages(projectRef) } "wait for storages to be indexed" in { @@ -91,9 +94,6 @@ abstract class StorageSpec extends BaseIntegrationSpec { "A json file" should { - val jsonFileContent = """{ "initial": ["is", "a", "test", "file"] }""" - val updatedJsonFileContent = """{ "updated": ["is", "a", "test", "file"] }""" - "be uploaded" in { deltaClient.uploadFile[Json]( s"/files/$projectRef/attachment.json?storage=nxv:$storageId", @@ -424,7 +424,7 @@ abstract class StorageSpec extends BaseIntegrationSpec { s"=?UTF-8?B?$encodedFilename?=" } - private def expectDownload( + protected def expectDownload( expectedFilename: String, expectedContentType: ContentType, expectedContent: String, From b30560f7f6856576e962fdf8b0b640bf3c5cb337 Mon Sep 17 00:00:00 2001 From: dantb Date: Wed, 29 Nov 2023 12:31:30 +0100 Subject: [PATCH 02/18] Accept list of files in copy route --- .../delta/plugins/storage/files/Files.scala | 12 +++++- .../files/model/CopyFileDestination.scala | 10 +---- .../files/routes/CopyFilePayload.scala | 36 ----------------- .../storage/files/routes/CopyFileSource.scala | 39 +++++++++++++++++++ .../files/routes/CopyFilesResponse.scala | 13 +++++++ .../storage/files/routes/FilesRoutes.scala | 38 +++++++----------- .../rdf/jsonld/encoder/JsonLdEncoder.scala | 2 + .../nexus/delta/sdk/model/ResourceF.scala | 8 +++- 8 files changed, 88 insertions(+), 70 deletions(-) delete mode 100644 delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/CopyFilePayload.scala create mode 100644 delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/CopyFileSource.scala create mode 100644 delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/CopyFilesResponse.scala diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala index 3f0dcc81ca..c143e3aae0 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala @@ -4,6 +4,7 @@ import akka.actor.typed.ActorSystem import akka.actor.{ActorSystem => ClassicActorSystem} import akka.http.scaladsl.model.ContentTypes.`application/octet-stream` import akka.http.scaladsl.model.{BodyPartEntity, ContentType, HttpEntity, Uri} +import cats.data.NonEmptyList import cats.effect.{Clock, IO} import cats.syntax.all._ import ch.epfl.bluebrain.nexus.delta.kernel.cache.LocalCache @@ -17,6 +18,7 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileCommand._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileEvent._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model._ +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.CopyFileSource import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.schemas.{files => fileSchema} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StoragesConfig.{RemoteDiskStorageConfig, StorageTypeConfig} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageRejection.{DifferentStorageType, InvalidStorageType, StorageFetchRejection, StorageIsDeprecated} @@ -195,6 +197,12 @@ final class Files( } yield res }.span("createLink") + def copyFiles( + source: CopyFileSource, + destination: CopyFileDestination + )(implicit c: Caller): IO[NonEmptyList[FileResource]] = + source.files.traverse(copyTo(_, destination)) + /** * Create a file from a source file potentially in a different organization * @param sourceId @@ -214,8 +222,8 @@ final class Files( _ <- IO.raiseUnless(space.exists(_ < file.attributes.bytes))( FileTooLarge(destStorage.storageValue.maxFileSize, space) ) - iri <- dest.fileId.fold(generateId(pc))(FileId(_, dest.project).expandIri(fetchContext.onCreate).map(_._1)) - destinationDesc <- FileDescription(dest.filename.getOrElse(file.attributes.filename), file.attributes.mediaType) + iri <- generateId(pc) + destinationDesc <- FileDescription(file.attributes.filename, file.attributes.mediaType) attributes <- CopyFile(destStorage, remoteDiskStorageClient).apply(file.attributes, destinationDesc).adaptError { case r: CopyFileRejection => CopyRejection(file.id, file.storage.iri, destStorage.id, r) } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/CopyFileDestination.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/CopyFileDestination.scala index 8cce82a5b6..03426b5b0b 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/CopyFileDestination.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/CopyFileDestination.scala @@ -5,23 +5,17 @@ import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef import ch.epfl.bluebrain.nexus.delta.sourcing.model.Tag.UserTag /** - * Details of the file we're creating in the copy + * Details for the files we're creating in the copy * * @param project * Orgnization and project for the new file - * @param fileId - * Optional identifier for the new file * @param storage * Optional storage for the new file which must have the same type as the source file's storage * @param tag * Optional tag to create the new file with - * @param filename - * Optional filename for the new file. If omitted, the source filename will be used */ final case class CopyFileDestination( project: ProjectRef, - fileId: Option[IdSegment], storage: Option[IdSegment], - tag: Option[UserTag], - filename: Option[String] + tag: Option[UserTag] ) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/CopyFilePayload.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/CopyFilePayload.scala deleted file mode 100644 index 6ef65bed38..0000000000 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/CopyFilePayload.scala +++ /dev/null @@ -1,36 +0,0 @@ -package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes - -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileId -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection.InvalidFileLookup -import ch.epfl.bluebrain.nexus.delta.sdk.model.IdSegment -import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef -import ch.epfl.bluebrain.nexus.delta.sourcing.model.Tag.UserTag -import io.circe.Decoder - -final case class CopyFilePayload( - destFilename: Option[String], - sourceProj: ProjectRef, - sourceFile: IdSegment, - sourceTag: Option[UserTag], - sourceRev: Option[Int] -) { - def toSourceFileId: Either[InvalidFileLookup, FileId] = (sourceTag, sourceRev) match { - case (Some(tag), None) => Right(FileId(sourceFile, tag, sourceProj)) - case (None, Some(rev)) => Right(FileId(sourceFile, rev, sourceProj)) - case (None, None) => Right(FileId(sourceFile, sourceProj)) - case (Some(_), Some(_)) => Left(InvalidFileLookup(sourceFile)) - } -} - -object CopyFilePayload { - - implicit val dec: Decoder[CopyFilePayload] = Decoder.instance { cur => - for { - destFilename <- cur.get[Option[String]]("destinationFilename") - sourceProj <- cur.get[ProjectRef]("sourceProjectRef") - sourceFileId <- cur.get[String]("sourceFileId").map(IdSegment(_)) - sourceTag <- cur.get[Option[UserTag]]("sourceTag") - sourceRev <- cur.get[Option[Int]]("sourceRev") - } yield CopyFilePayload(destFilename, sourceProj, sourceFileId, sourceTag, sourceRev) - } -} diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/CopyFileSource.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/CopyFileSource.scala new file mode 100644 index 0000000000..afe5ee50bc --- /dev/null +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/CopyFileSource.scala @@ -0,0 +1,39 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes + +import cats.data.NonEmptyList +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileId +import ch.epfl.bluebrain.nexus.delta.sdk.model.IdSegment +import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef +import ch.epfl.bluebrain.nexus.delta.sourcing.model.Tag.UserTag +import io.circe.{Decoder, DecodingFailure, Json} + +final case class CopyFileSource( + project: ProjectRef, + files: NonEmptyList[FileId] +) + +object CopyFileSource { + + implicit val dec: Decoder[CopyFileSource] = Decoder.instance { cur => + def parseSingle(j: Json, proj: ProjectRef): Decoder.Result[FileId] = + for { + sourceFile <- j.hcursor.get[String]("sourceFileId").map(IdSegment(_)) + sourceTag <- j.hcursor.get[Option[UserTag]]("sourceTag") + sourceRev <- j.hcursor.get[Option[Int]]("sourceRev") + fileId <- (sourceTag, sourceRev) match { + case (Some(tag), None) => Right(FileId(sourceFile, tag, proj)) + case (None, Some(rev)) => Right(FileId(sourceFile, rev, proj)) + case (None, None) => Right(FileId(sourceFile, proj)) + case (Some(_), Some(_)) => + Left( + DecodingFailure("Tag and revision cannot be simultaneously present for source file lookup", Nil) + ) + } + } yield fileId + + for { + sourceProj <- cur.get[ProjectRef]("sourceProjectRef") + files <- cur.get[NonEmptyList[Json]]("files").flatMap(_.traverse(parseSingle(_, sourceProj))) + } yield CopyFileSource(sourceProj, files) + } +} diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/CopyFilesResponse.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/CopyFilesResponse.scala new file mode 100644 index 0000000000..1e801ba2cc --- /dev/null +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/CopyFilesResponse.scala @@ -0,0 +1,13 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes + +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.FileResource + +final case class FailureSummary( + source: CopyFileSource, + reason: String // TODO use ADT +) + +final case class CopyFilesResponse( + successes: List[FileResource], + failures: List[FailureSummary] +) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutes.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutes.scala index f44a284601..4545f4ea9e 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutes.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutes.scala @@ -5,7 +5,7 @@ import akka.http.scaladsl.model.Uri.Path import akka.http.scaladsl.model.headers.Accept import akka.http.scaladsl.model.{ContentType, MediaRange} import akka.http.scaladsl.server._ -import cats.data.EitherT +import cats.data.{EitherT, NonEmptyList} import cats.effect.IO import cats.syntax.all._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection._ @@ -15,6 +15,7 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.FilesRoutes._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{schemas, FileResource, Files} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StoragesConfig.StorageTypeConfig import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.RemoteContextResolution +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.encoder.JsonLdEncoder import ch.epfl.bluebrain.nexus.delta.rdf.utils.JsonKeyOrdering import ch.epfl.bluebrain.nexus.delta.sdk._ import ch.epfl.bluebrain.nexus.delta.sdk.acls.AclCheck @@ -28,7 +29,6 @@ import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller import ch.epfl.bluebrain.nexus.delta.sdk.implicits._ import ch.epfl.bluebrain.nexus.delta.sdk.model.routes.Tag import ch.epfl.bluebrain.nexus.delta.sdk.model.{BaseUri, IdSegment} -import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef import ch.epfl.bluebrain.nexus.delta.sourcing.model.Tag.UserTag import io.circe.Decoder import io.circe.generic.extras.Configuration @@ -69,6 +69,9 @@ final class FilesRoutes( import baseUri.prefixSegment import schemeDirectives._ + implicit val nelEnc: JsonLdEncoder[NonEmptyList[FileResource]] = + JsonLdEncoder.computeFromCirce[NonEmptyList[FileResource]](Files.context) + def routes: Route = (baseUriPrefix(baseUri.prefix) & replaceUri("files", schemas.files)) { pathPrefix("files") { @@ -94,11 +97,10 @@ final class FilesRoutes( .attemptNarrow[FileRejection] ) }, - // Create a file by copying from another project, without id segment - entity(as[CopyFilePayload]) { c: CopyFilePayload => - val copyTo = CopyFileDestination(projectRef, None, storage, tag, c.destFilename) - - emit(Created, copyFile(projectRef, mode, c, copyTo)) + // Bulk create files by copying from another project + entity(as[CopyFileSource]) { c: CopyFileSource => + val copyTo = CopyFileDestination(projectRef, storage, tag) + emit(Created, copyFile(mode, c, copyTo)) }, // Create a file without id segment extractRequestEntity { entity => @@ -118,15 +120,6 @@ final class FilesRoutes( concat( (put & pathEndOrSingleSlash) { concat( - // Create a file by copying from another project - parameters("storage".as[IdSegment].?, "tag".as[UserTag].?) { case (destStorage, destTag) => - entity(as[CopyFilePayload]) { c: CopyFilePayload => - val copyTo = - CopyFileDestination(projectRef, Some(id), destStorage, destTag, c.destFilename) - - emit(Created, copyFile(projectRef, mode, c, copyTo)) - } - }, parameters("rev".as[Int], "storage".as[IdSegment].?, "tag".as[UserTag].?) { case (rev, storage, tag) => concat( @@ -254,14 +247,13 @@ final class FilesRoutes( } } - private def copyFile(projectRef: ProjectRef, mode: IndexingMode, c: CopyFilePayload, copyTo: CopyFileDestination)( - implicit caller: Caller - ): IO[Either[FileRejection, FileResource]] = + private def copyFile(mode: IndexingMode, c: CopyFileSource, copyTo: CopyFileDestination)(implicit + caller: Caller + ): IO[Either[FileRejection, NonEmptyList[FileResource]]] = (for { - _ <- EitherT.right(aclCheck.authorizeForOr(c.sourceProj, Read)(AuthorizationFailed(c.sourceProj.project, Read))) - sourceFileId <- EitherT.fromEither[IO](c.toSourceFileId) - result <- EitherT(files.copyTo(sourceFileId, copyTo).attemptNarrow[FileRejection]) - _ <- EitherT.right[FileRejection](index(projectRef, result, mode)) + _ <- EitherT.right(aclCheck.authorizeForOr(c.project, Read)(AuthorizationFailed(c.project.project, Read))) + result <- EitherT(files.copyFiles(c, copyTo).attemptNarrow[FileRejection]) + _ <- EitherT.right[FileRejection](result.traverse(index(copyTo.project, _, mode))) } yield result).value def fetch(id: FileId)(implicit caller: Caller): Route = diff --git a/delta/rdf/src/main/scala/ch/epfl/bluebrain/nexus/delta/rdf/jsonld/encoder/JsonLdEncoder.scala b/delta/rdf/src/main/scala/ch/epfl/bluebrain/nexus/delta/rdf/jsonld/encoder/JsonLdEncoder.scala index 786fbb7390..b6ab2aefbe 100644 --- a/delta/rdf/src/main/scala/ch/epfl/bluebrain/nexus/delta/rdf/jsonld/encoder/JsonLdEncoder.scala +++ b/delta/rdf/src/main/scala/ch/epfl/bluebrain/nexus/delta/rdf/jsonld/encoder/JsonLdEncoder.scala @@ -97,6 +97,8 @@ trait JsonLdEncoder[A] { object JsonLdEncoder { + def apply[A](enc: JsonLdEncoder[A]): JsonLdEncoder[A] = enc + private def randomRootNode[A]: A => BNode = (_: A) => BNode.random /** diff --git a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/model/ResourceF.scala b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/model/ResourceF.scala index 0f72f1c6ee..66884ecc95 100644 --- a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/model/ResourceF.scala +++ b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/model/ResourceF.scala @@ -177,13 +177,19 @@ object ResourceF { implicit private def metadataJsonLdEncoder(implicit base: BaseUri): JsonLdEncoder[ResourceMetadata] = JsonLdEncoder.computeFromCirce(BNode.random, ContextValue(contexts.metadata)) - implicit def resourceFEncoder[A: Encoder.AsObject](implicit base: BaseUri): Encoder.AsObject[ResourceF[A]] = + implicit def resourceFEncoderObj[A: Encoder.AsObject](implicit base: BaseUri): Encoder.AsObject[ResourceF[A]] = Encoder.encodeJsonObject.contramapObject { r => ResourceIdAndTypes(r.resolvedId, r.types).asJsonObject deepMerge r.value.asJsonObject deepMerge ResourceMetadata(r).asJsonObject } + implicit def resourceFEncoder[A: Encoder](implicit base: BaseUri): Encoder[ResourceF[A]] = Encoder.instance { r => + ResourceIdAndTypes(r.resolvedId, r.types).asJson deepMerge + r.value.asJson deepMerge + ResourceMetadata(r).asJson + } + final private case class ResourceIdAndTypes(resolvedId: Iri, types: Set[Iri]) implicit private val idAndTypesEncoder: Encoder.AsObject[ResourceIdAndTypes] = From f2c6b525397854b56e92d59df4b487fd1dffa585 Mon Sep 17 00:00:00 2001 From: dantb Date: Mon, 4 Dec 2023 21:55:19 +0100 Subject: [PATCH 03/18] Move CopyFiles to kernel, reuse for delta disk storage, validate storage space during copy --- build.sbt | 1 + .../nexus/delta/kernel/utils}/CopyFiles.scala | 16 +- .../delta/kernel/utils/CopyFilesSuite.scala | 7 +- .../delta/plugins/storage/files/Files.scala | 108 +++++++++---- .../storage/files/model/CopyFileDetails.scala | 6 + .../storages/operations/CopyFile.scala | 7 +- .../operations/disk/DiskStorageCopyFile.scala | 52 ++++--- .../operations/disk/DiskStorageSaveFile.scala | 14 +- .../remote/RemoteDiskStorageCopyFile.scala | 38 +++-- .../client/RemoteDiskStorageClient.scala | 43 +++--- .../plugins/storage/files/FileFixtures.scala | 2 + .../plugins/storage/files/FilesSpec.scala | 143 +++++++++++++----- .../epfl/bluebrain/nexus/storage/Main.scala | 3 +- .../nexus/storage/StorageError.scala | 5 +- .../bluebrain/nexus/storage/Storages.scala | 10 +- .../nexus/storage/files/CopyBetween.scala | 12 -- .../nexus/storage/DiskStorageSpec.scala | 3 +- 17 files changed, 321 insertions(+), 149 deletions(-) rename {storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/files => delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils}/CopyFiles.scala (75%) rename storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/files/CopyFileSuite.scala => delta/kernel/src/test/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/CopyFilesSuite.scala (96%) create mode 100644 delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/CopyFileDetails.scala delete mode 100644 storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/files/CopyBetween.scala diff --git a/build.sbt b/build.sbt index 48ada3f12b..d80858ad8a 100755 --- a/build.sbt +++ b/build.sbt @@ -223,6 +223,7 @@ lazy val kernel = project scalaTest % Test ), addCompilerPlugin(kindProjector), + addCompilerPlugin(betterMonadicFor), coverageFailOnMinimum := false ) diff --git a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/files/CopyFiles.scala b/delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/CopyFiles.scala similarity index 75% rename from storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/files/CopyFiles.scala rename to delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/CopyFiles.scala index ed6eddb780..a302cec1d4 100644 --- a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/files/CopyFiles.scala +++ b/delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/CopyFiles.scala @@ -1,18 +1,24 @@ -package ch.epfl.bluebrain.nexus.storage.files +package ch.epfl.bluebrain.nexus.delta.kernel.utils import cats.data.NonEmptyList import cats.effect.{IO, Ref} import cats.implicits._ -import ch.epfl.bluebrain.nexus.storage.StorageError.CopyOperationFailed +import ch.epfl.bluebrain.nexus.delta.kernel.error.Rejection import fs2.io.file.{CopyFlag, CopyFlags, Files, Path} trait CopyFiles { - def copyValidated(files: NonEmptyList[ValidatedCopyFile]): IO[Unit] + def copyAll(files: NonEmptyList[CopyBetween]): IO[Unit] +} + +final case class CopyBetween(source: Path, destination: Path) + +final case class CopyOperationFailed(failingCopy: CopyBetween) extends Rejection { + override def reason: String = + s"Copy operation failed from source ${failingCopy.source} to destination ${failingCopy.destination}." } object CopyFiles { - def mk(): CopyFiles = files => - copyAll(files.map(v => CopyBetween(Path.fromNioPath(v.absSourcePath), Path.fromNioPath(v.absDestPath)))) + def mk(): CopyFiles = files => copyAll(files) def copyAll(files: NonEmptyList[CopyBetween]): IO[Unit] = Ref.of[IO, Option[CopyOperationFailed]](None).flatMap { errorRef => diff --git a/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/files/CopyFileSuite.scala b/delta/kernel/src/test/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/CopyFilesSuite.scala similarity index 96% rename from storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/files/CopyFileSuite.scala rename to delta/kernel/src/test/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/CopyFilesSuite.scala index d962ea3709..ba426068bc 100644 --- a/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/files/CopyFileSuite.scala +++ b/delta/kernel/src/test/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/CopyFilesSuite.scala @@ -1,10 +1,9 @@ -package ch.epfl.bluebrain.nexus.storage.files +package ch.epfl.bluebrain.nexus.delta.kernel.utils import cats.data.NonEmptyList import cats.effect.IO import cats.syntax.all._ -import ch.epfl.bluebrain.nexus.storage.StorageError.CopyOperationFailed -import ch.epfl.bluebrain.nexus.storage.files.CopyFiles.parent +import ch.epfl.bluebrain.nexus.delta.kernel.utils.CopyFiles.parent import fs2.io.file.PosixPermission._ import fs2.io.file._ import munit.CatsEffectSuite @@ -12,7 +11,7 @@ import munit.catseffect.IOFixture import java.util.UUID -class CopyFileSuite extends CatsEffectSuite { +class CopyFilesSuite extends CatsEffectSuite { val myFixture: IOFixture[Path] = ResourceSuiteLocalFixture("create-temp-dir-fixture", Files[IO].tempDirectory) override def munitFixtures: List[IOFixture[Path]] = List(myFixture) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala index c143e3aae0..2d0893033a 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala @@ -23,7 +23,7 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.schemas.{files => fil import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StoragesConfig.{RemoteDiskStorageConfig, StorageTypeConfig} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageRejection.{DifferentStorageType, InvalidStorageType, StorageFetchRejection, StorageIsDeprecated} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{DigestAlgorithm, Storage, StorageRejection, StorageType} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.{CopyFileRejection, FetchAttributeRejection, FetchFileRejection, SaveFileRejection} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.{FetchAttributeRejection, FetchFileRejection, SaveFileRejection} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.RemoteDiskStorageClient import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{Storages, StoragesStatistics} @@ -199,37 +199,83 @@ final class Files( def copyFiles( source: CopyFileSource, - destination: CopyFileDestination - )(implicit c: Caller): IO[NonEmptyList[FileResource]] = - source.files.traverse(copyTo(_, destination)) - - /** - * Create a file from a source file potentially in a different organization - * @param sourceId - * File lookup id for the source file - * @param dest - * Project, storage and file details for the file we're creating - */ - def copyTo( - sourceId: FileId, dest: CopyFileDestination - )(implicit c: Caller): IO[FileResource] = { + )(implicit c: Caller): IO[NonEmptyList[FileResource]] = { for { - file <- fetchSourceFile(sourceId) (pc, destStorageRef, destStorage) <- fetchDestinationStorage(dest) - _ <- validateStorageTypeForCopy(file.storageType, destStorage) - space <- fetchStorageAvailableSpace(destStorage) - _ <- IO.raiseUnless(space.exists(_ < file.attributes.bytes))( - FileTooLarge(destStorage.storageValue.maxFileSize, space) - ) - iri <- generateId(pc) - destinationDesc <- FileDescription(file.attributes.filename, file.attributes.mediaType) - attributes <- CopyFile(destStorage, remoteDiskStorageClient).apply(file.attributes, destinationDesc).adaptError { - case r: CopyFileRejection => CopyRejection(file.id, file.storage.iri, destStorage.id, r) - } - res <- eval(CreateFile(iri, dest.project, destStorageRef, destStorage.tpe, attributes, c.subject, dest.tag)) - } yield res - }.span("copyFile") + copyDetails <- source.files.traverse(fetchCopyDetails(destStorage, _)) + _ <- validateSpaceOnStorage(destStorage, copyDetails) + destFilesAttributes <- CopyFile(destStorage, remoteDiskStorageClient).apply(copyDetails) + fileResources <- evalCreateCommands(pc, dest, destStorageRef, destStorage.tpe, destFilesAttributes) + } yield fileResources + } + + private def evalCreateCommands( + pc: ProjectContext, + dest: CopyFileDestination, + destStorageRef: ResourceRef.Revision, + destStorageTpe: StorageType, + destFilesAttributes: NonEmptyList[FileAttributes] + )(implicit c: Caller): IO[NonEmptyList[FileResource]] = + destFilesAttributes.traverse { destFileAttributes => + for { + iri <- generateId(pc) + command = CreateFile(iri, dest.project, destStorageRef, destStorageTpe, destFileAttributes, c.subject, dest.tag) + resource <- eval(command).onError { e => + logger.error(e)( + s"Failed to save event during file copy, saved file must be manually deleted: $command" + ) + } + } yield resource + } + + private def validateSpaceOnStorage(destStorage: Storage, copyDetails: NonEmptyList[CopyFileDetails]): IO[Unit] = for { + space <- fetchStorageAvailableSpace(destStorage) + allSizes = copyDetails.map(_.sourceAttributes.bytes) + maxSize = destStorage.storageValue.maxFileSize + _ <- IO.raiseWhen(allSizes.exists(_ > maxSize))(FileTooLarge(maxSize, space)) + totalSize = allSizes.toList.sum + _ <- IO.raiseWhen(space.exists(_ < totalSize))(FileTooLarge(maxSize, space)) + } yield () + + private def fetchCopyDetails(destStorage: Storage, fileId: FileId)(implicit c: Caller): IO[CopyFileDetails] = + for { + file <- fetchSourceFile(fileId) + _ <- validateStorageTypeForCopy(file.storageType, destStorage) + destinationDesc <- FileDescription(file.attributes.filename, file.attributes.mediaType) + } yield CopyFileDetails(destinationDesc, file.attributes) + +// /** +// * Create a file from a source file potentially in a different organization +// * +// * @param sourceId +// * File lookup id for the source file +// * @param dest +// * Project, storage and file details for the file we're creating +// */ +// def copyTo( +// sourceId: FileId, +// dest: CopyFileDestination +// )(implicit c: Caller): IO[FileResource] = { +// for { +// file <- fetchSourceFile(sourceId) +// (pc, destStorageRef, destStorage) <- fetchDestinationStorage(dest) +// _ <- validateStorageTypeForCopy(file.storageType, destStorage) +// space <- fetchStorageAvailableSpace(destStorage) +// _ <- IO.raiseUnless(space.exists(_ < file.attributes.bytes))( +// FileTooLarge(destStorage.storageValue.maxFileSize, space) +// ) +// iri <- generateId(pc) +// destinationDesc <- FileDescription(file.attributes.filename, file.attributes.mediaType) +// attributes <- CopyFile(destStorage, remoteDiskStorageClient).apply(file.attributes, destinationDesc).adaptError { +// case r: CopyFileRejection => CopyRejection(file.id, file.storage.iri, destStorage.id, r) +// } +// res <- eval(CreateFile(iri, dest.project, destStorageRef, destStorage.tpe, attributes, c.subject, dest.tag)) +// } yield res +// }.span("copyFile") + +// private def doCopy(files: NonEmptyList[CopyFileDetails]): IO[NonEmptyList[FileAttributes]] = +// private def fetchSourceFile(id: FileId)(implicit c: Caller) = for { @@ -238,7 +284,9 @@ final class Files( _ <- validateAuth(id.project, sourceStorage.value.storageValue.readPermission) } yield file.value - private def fetchDestinationStorage(dest: CopyFileDestination)(implicit c: Caller) = + private def fetchDestinationStorage( + dest: CopyFileDestination + )(implicit c: Caller): IO[(ProjectContext, ResourceRef.Revision, Storage)] = for { pc <- fetchContext.onCreate(dest.project) (destStorageRef, destStorage) <- fetchActiveStorage(dest.storage, dest.project, pc) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/CopyFileDetails.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/CopyFileDetails.scala new file mode 100644 index 0000000000..a3b31c844e --- /dev/null +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/CopyFileDetails.scala @@ -0,0 +1,6 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model + +final case class CopyFileDetails( + destinationDesc: FileDescription, + sourceAttributes: FileAttributes +) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/CopyFile.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/CopyFile.scala index c23c4976c7..64c81cdf90 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/CopyFile.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/CopyFile.scala @@ -1,13 +1,14 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations +import cats.data.NonEmptyList import cats.effect.IO -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes, FileDescription} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{CopyFileDetails, FileAttributes} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{Storage, StorageType} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.CopyFileRejection import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.RemoteDiskStorageClient trait CopyFile { - def apply(source: FileAttributes, dest: FileDescription): IO[FileAttributes] + def apply(copyDetails: NonEmptyList[CopyFileDetails]): IO[NonEmptyList[FileAttributes]] } object CopyFile { @@ -20,6 +21,6 @@ object CopyFile { } private def unsupported(storageType: StorageType): CopyFile = - (_, _) => IO.raiseError(CopyFileRejection.UnsupportedOperation(storageType)) + _ => IO.raiseError(CopyFileRejection.UnsupportedOperation(storageType)) } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageCopyFile.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageCopyFile.scala index fbe4ffe1dc..cd1674b8c8 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageCopyFile.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageCopyFile.scala @@ -1,32 +1,44 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk import akka.http.scaladsl.model.Uri +import cats.data.NonEmptyList import cats.effect.IO -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes, FileDescription} +import ch.epfl.bluebrain.nexus.delta.kernel.utils.{CopyBetween, CopyFiles} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{CopyFileDetails, FileAttributes} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.DiskStorage import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.CopyFile -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.DiskStorageSaveFile.initLocation +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.DiskStorageSaveFile.computeLocation +import fs2.io.file.Path import java.net.URI -import java.nio.file.{Paths, StandardCopyOption} -import scala.annotation.nowarn +import java.nio.file.Paths class DiskStorageCopyFile(storage: DiskStorage) extends CopyFile { - @nowarn - override def apply(source: FileAttributes, dest: FileDescription): IO[FileAttributes] = { - val sourcePath = Paths.get(URI.create(s"file://${source.location.path}")) - for { - (destPath, destRelativePath) <- initLocation(storage.project, storage.value, dest.uuid, dest.filename) - _ <- fs2.io.file.copy[IO](sourcePath, destPath, Seq(StandardCopyOption.COPY_ATTRIBUTES)) - } yield FileAttributes( - uuid = dest.uuid, - location = Uri(destPath.toUri.toString), - path = Uri.Path(destRelativePath.toString), - filename = dest.filename, - mediaType = source.mediaType, - bytes = source.bytes, - digest = source.digest, - origin = source.origin - ) + override def apply(details: NonEmptyList[CopyFileDetails]): IO[NonEmptyList[FileAttributes]] = { + details + .traverse { copyFile => + val dest = copyFile.destinationDesc + val sourcePath = Paths.get(URI.create(s"file://${copyFile.sourceAttributes.location.path}")) + for { + (destPath, destRelativePath) <- computeLocation(storage.project, storage.value, dest.uuid, dest.filename) + } yield sourcePath -> FileAttributes( + uuid = dest.uuid, + location = Uri(destPath.toUri.toString), + path = Uri.Path(destRelativePath.toString), + filename = dest.filename, + mediaType = copyFile.sourceAttributes.mediaType, + bytes = copyFile.sourceAttributes.bytes, + digest = copyFile.sourceAttributes.digest, + origin = copyFile.sourceAttributes.origin + ) + } + .flatMap { destinationAttributesBySourcePath => + val paths = destinationAttributesBySourcePath.map { case (sourcePath, destAttr) => + CopyBetween(Path.fromNioPath(sourcePath), Path(destAttr.location.toString())) + } + CopyFiles.copyAll(paths).as { + destinationAttributesBySourcePath.map { case (_, destAttr) => destAttr } + } + } } } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageSaveFile.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageSaveFile.scala index cae5520aae..ef428d3d27 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageSaveFile.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageSaveFile.scala @@ -70,13 +70,23 @@ object DiskStorageSaveFile { disk: DiskStorageValue, uuid: UUID, filename: String + ): IO[(Path, Path)] = + for { + (resolved, relative) <- computeLocation(project, disk, uuid, filename) + dir = resolved.getParent + _ <- IO.delay(Files.createDirectories(dir)).adaptError(couldNotCreateDirectory(dir, _)) + } yield resolved -> relative + + def computeLocation( + project: ProjectRef, + disk: DiskStorageValue, + uuid: UUID, + filename: String ): IO[(Path, Path)] = { val relativePath = intermediateFolders(project, uuid, filename) for { relative <- IO.delay(Paths.get(relativePath)).adaptError(wrongPath(relativePath, _)) resolved <- IO.delay(disk.volume.value.resolve(relative)).adaptError(wrongPath(relativePath, _)) - dir = resolved.getParent - _ <- IO.delay(Files.createDirectories(dir)).adaptError(couldNotCreateDirectory(dir, _)) } yield resolved -> relative } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageCopyFile.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageCopyFile.scala index 71236dbae9..4d6ddc7ec8 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageCopyFile.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageCopyFile.scala @@ -1,8 +1,9 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote import akka.http.scaladsl.model.Uri +import cats.data.NonEmptyList import cats.effect.IO -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes, FileDescription} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{CopyFileDetails, FileAttributes} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.RemoteDiskStorage import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.CopyFile import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.SaveFile.intermediateFolders @@ -13,21 +14,28 @@ class RemoteDiskStorageCopyFile( client: RemoteDiskStorageClient ) extends CopyFile { - def apply(source: FileAttributes, description: FileDescription): IO[FileAttributes] = { - val destinationPath = Uri.Path(intermediateFolders(storage.project, description.uuid, description.filename)) - client.copyFile(storage.value.folder, source.location.path, destinationPath)(storage.value.endpoint).as { - FileAttributes( - uuid = description.uuid, - location = source.location, // TODO what's the destination absolute path? - path = destinationPath, - filename = description.filename, - mediaType = description.mediaType, - bytes = source.bytes, - digest = source.digest, - origin = source.origin - ) + override def apply(copyDetails: NonEmptyList[CopyFileDetails]): IO[NonEmptyList[FileAttributes]] = { + val thing = copyDetails.map { cd => + val destinationPath = + Uri.Path(intermediateFolders(storage.project, cd.destinationDesc.uuid, cd.destinationDesc.filename)) + val sourcePath = cd.sourceAttributes.location + (sourcePath, destinationPath) } - } + client.copyFile(storage.value.folder, thing)(storage.value.endpoint).map { destPaths => + copyDetails.zip(destPaths).map { case (cd, destinationPath) => + FileAttributes( + uuid = cd.destinationDesc.uuid, + location = destinationPath, + path = destinationPath.path, + filename = cd.destinationDesc.filename, + mediaType = cd.destinationDesc.mediaType, + bytes = cd.sourceAttributes.bytes, + digest = cd.sourceAttributes.digest, + origin = cd.sourceAttributes.origin + ) + } + } + } } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/RemoteDiskStorageClient.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/RemoteDiskStorageClient.scala index cf2336fbc7..7f33a75e7c 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/RemoteDiskStorageClient.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/RemoteDiskStorageClient.scala @@ -2,17 +2,19 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote import akka.actor.ActorSystem import akka.http.scaladsl.client.RequestBuilding._ -import akka.http.scaladsl.model.BodyPartEntity import akka.http.scaladsl.model.Multipart.FormData import akka.http.scaladsl.model.Multipart.FormData.BodyPart import akka.http.scaladsl.model.StatusCodes._ import akka.http.scaladsl.model.Uri.Path +import akka.http.scaladsl.model.{BodyPartEntity, Uri} +import cats.data.NonEmptyList import cats.effect.IO import cats.implicits._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.FetchFileRejection.UnexpectedFetchError import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.MoveFileRejection.UnexpectedMoveError import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.{FetchFileRejection, MoveFileRejection, SaveFileRejection} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.model.RemoteDiskStorageFileAttributes +import ch.epfl.bluebrain.nexus.delta.rdf.implicits.uriDecoder import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.JsonLdContext.keywords import ch.epfl.bluebrain.nexus.delta.sdk.AkkaSource import ch.epfl.bluebrain.nexus.delta.sdk.auth.{AuthTokenProvider, Credentials} @@ -187,25 +189,30 @@ final class RemoteDiskStorageClient(client: HttpClient, getAuthToken: AuthTokenP */ def copyFile( bucket: Label, - sourceRelativePath: Path, - destRelativePath: Path - )(implicit baseUri: BaseUri): IO[Unit] = { + files: NonEmptyList[(Uri, Path)] + )(implicit baseUri: BaseUri): IO[NonEmptyList[Uri]] = { getAuthToken(credentials).flatMap { authToken => - val endpoint = baseUri.endpoint / "buckets" / bucket.value / "files" / destRelativePath - val payload = Json.obj("source" -> sourceRelativePath.toString.asJson) + val endpoint = baseUri.endpoint / "buckets" / bucket.value / "files" + val payload = files.map { case (source, dest) => + Json.obj("source" := source.toString(), "destination" := dest.toString()) + }.asJson + + implicit val dec: Decoder[NonEmptyList[Uri]] = Decoder[NonEmptyList[Json]].emap { nel => + nel.traverse(_.hcursor.get[Uri]("absoluteDestinationLocation").leftMap(_.toString())) + } client - .discardBytes(Post(endpoint, payload).withCredentials(authToken), ()) - .adaptError { - // TODO update error - case error @ HttpClientStatusError(_, `NotFound`, _) if !bucketNotFoundType(error) => - MoveFileRejection.FileNotFound(sourceRelativePath.toString) - case error @ HttpClientStatusError(_, `BadRequest`, _) if pathContainsLinksType(error) => - MoveFileRejection.PathContainsLinks(destRelativePath.toString) - case HttpClientStatusError(_, `Conflict`, _) => - MoveFileRejection.ResourceAlreadyExists(destRelativePath.toString) - case error: HttpClientError => - UnexpectedMoveError(sourceRelativePath.toString, destRelativePath.toString, error.asString) - } + .fromJsonTo[NonEmptyList[Uri]](Post(endpoint, payload).withCredentials(authToken)) + // TODO update error +// .adaptError { +// case error @ HttpClientStatusError(_, `NotFound`, _) if !bucketNotFoundType(error) => +// MoveFileRejection.FileNotFound(sourceRelativePath.toString) +// case error @ HttpClientStatusError(_, `BadRequest`, _) if pathContainsLinksType(error) => +// MoveFileRejection.PathContainsLinks(destRelativePath.toString) +// case HttpClientStatusError(_, `Conflict`, _) => +// MoveFileRejection.ResourceAlreadyExists(destRelativePath.toString) +// case error: HttpClientError => +// UnexpectedMoveError(sourceRelativePath.toString, destRelativePath.toString, error.asString) +// } } } diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FileFixtures.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FileFixtures.scala index 219d674c4d..35043e76c8 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FileFixtures.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FileFixtures.scala @@ -25,6 +25,8 @@ trait FileFixtures extends EitherValues { val uuid = UUID.fromString("8249ba90-7cc6-4de5-93a1-802c04200dcc") val uuid2 = UUID.fromString("12345678-7cc6-4de5-93a1-802c04200dcc") + val uuid3 = UUID.randomUUID() + val uuid4 = UUID.randomUUID() val uuidOrg2 = UUID.fromString("66666666-7cc6-4de5-93a1-802c04200dcc") val ref = Ref.of[IO, UUID](uuid).unsafeRunSync() implicit val uuidF: UUIDF = UUIDF.fromRef(ref) diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala index 6ed0ab682c..19f4272c04 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala @@ -5,13 +5,16 @@ import akka.actor.{typed, ActorSystem} import akka.http.scaladsl.model.ContentTypes.`text/plain(UTF-8)` import akka.http.scaladsl.model.Uri import akka.testkit.TestKit +import cats.data.NonEmptyList import cats.effect.IO +import cats.effect.unsafe.implicits.global import ch.epfl.bluebrain.nexus.delta.kernel.http.MediaTypeDetectorConfig import ch.epfl.bluebrain.nexus.delta.plugins.storage.RemoteContextResolutionFixture import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest.NotComputedDigest import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes.FileAttributesOrigin.Storage import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{CopyFileDestination, FileAttributes, FileId, FileRejection} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.CopyFileSource import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageRejection.{DifferentStorageType, StorageNotFound} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageType.{RemoteDiskStorage => RemoteStorageType} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{StorageRejection, StorageStatEntry, StorageType} @@ -44,6 +47,7 @@ import org.scalatest.concurrent.Eventually import org.scalatest.{Assertion, DoNotDiscover} import java.net.URLDecoder +import java.util.UUID @DoNotDiscover class FilesSpec(docker: RemoteStorageDocker) @@ -87,6 +91,8 @@ class FilesSpec(docker: RemoteStorageDocker) val diskId: IdSegment = nxv + "disk" val diskRev = ResourceRef.Revision(iri"$diskId?rev=1", diskIdIri, 1) + val smallDiskId: IdSegment = nxv + "smalldisk" + val storageIri = nxv + "other-storage" val storage: IdSegment = nxv + "other-storage" @@ -107,8 +113,10 @@ class FilesSpec(docker: RemoteStorageDocker) remoteDisk = Some(config.remoteDisk.value.copy(defaultMaxFileSize = 500)) ) - val storageStatistics: StoragesStatistics = - (_, _) => IO.pure { StorageStatEntry(10L, 100L) } + val storageStatistics: StoragesStatistics = { + case (`smallDiskId`, _) => IO.pure { StorageStatEntry(10L, 0L) } + case (_, _) => IO.pure { StorageStatEntry(10L, 100L) } + } lazy val storages: Storages = Storages( fetchContext.mapRejection(StorageRejection.ProjectContextRejection), @@ -443,51 +451,117 @@ class FilesSpec(docker: RemoteStorageDocker) "copying a file" should { "succeed from disk storage based on a tag" in { - val newFileId = genString() - val destination = CopyFileDestination(projectRefOrg2, Some(newFileId), None, None, None) - val expectedFilename = "myfile.txt" - val expectedAttr = attributes(filename = expectedFilename, projRef = projectRefOrg2) - val expected = mkResource(nxv + newFileId, projectRefOrg2, diskRev, expectedAttr) - - val actual = files.copyTo(FileId("file1", tag, projectRef), destination).accepted - actual shouldEqual expected - - val fetched = files.fetch(FileId(newFileId, projectRefOrg2)).accepted - fetched shouldEqual expected + // TODO: adding uuids whenever we want a new independent test is not sustainable. If we truly want to test this every + // time we should generate a new "Files" with a new UUIDF. + // Alternatively we could normalise the expected values to not care about any generated Ids + val newFileUuid = UUID.randomUUID() + withUUIDF(newFileUuid) { + val source = CopyFileSource(projectRef, NonEmptyList.of(FileId("file1", tag, projectRef))) + val destination = CopyFileDestination(projectRefOrg2, Some(diskId), None) + + val expectedDestId = project2.base.iri / newFileUuid.toString + val expectedFilename = "myfile.txt" + val expectedAttr = attributes(filename = expectedFilename, projRef = projectRefOrg2, id = newFileUuid) + val expected = mkResource(expectedDestId, projectRefOrg2, diskRev, expectedAttr) + + val actual = files.copyFiles(source, destination).unsafeRunSync() + actual shouldEqual NonEmptyList.of(expected) + + val fetched = files.fetch(FileId(newFileUuid.toString, projectRefOrg2)).accepted + fetched shouldEqual expected + } } "succeed from disk storage based on a rev and should tag the new file" in { - val (newFileId, newTag) = (genString(), UserTag.unsafe(genString())) - val destination = - CopyFileDestination(projectRefOrg2, Some(newFileId), None, Some(newTag), None) - val expectedFilename = "file.txt" - val expectedAttr = attributes(filename = expectedFilename, projRef = projectRefOrg2) - val expected = mkResource(nxv + newFileId, projectRefOrg2, diskRev, expectedAttr, tags = Tags(newTag -> 1)) - - val actual = files.copyTo(FileId("file1", 2, projectRef), destination).accepted - actual shouldEqual expected + val newFileUuid = UUID.randomUUID() + withUUIDF(newFileUuid) { + val source = CopyFileSource(projectRef, NonEmptyList.of(FileId("file1", 2, projectRef))) + val newTag = UserTag.unsafe(genString()) + val destination = CopyFileDestination(projectRefOrg2, Some(diskId), Some(newTag)) + + val expectedDestId = project2.base.iri / newFileUuid.toString + val expectedFilename = "file.txt" + val expectedAttr = attributes(filename = expectedFilename, projRef = projectRefOrg2, id = newFileUuid) + val expected = mkResource(expectedDestId, projectRefOrg2, diskRev, expectedAttr, tags = Tags(newTag -> 1)) + + val actual = files.copyFiles(source, destination).accepted + actual shouldEqual NonEmptyList.of(expected) + + val fetchedByTag = files.fetch(FileId(newFileUuid.toString, newTag, projectRefOrg2)).accepted + fetchedByTag shouldEqual expected + } + } + + "succeed from remote storage based on latest" in { + val newFileUuid = UUID.randomUUID() + withUUIDF(newFileUuid) { + val source = CopyFileSource(projectRef, NonEmptyList.of(FileId("file1", tag, projectRef))) + val destination = CopyFileDestination(projectRefOrg2, Some(remoteId), None) + + val expectedDestId = project2.base.iri / newFileUuid.toString + val expectedFilename = "myfile.txt" + val expectedAttr = attributes(filename = expectedFilename, projRef = projectRefOrg2, id = newFileUuid) + val expected = mkResource(expectedDestId, projectRefOrg2, diskRev, expectedAttr) + + val actual = files.copyFiles(source, destination).unsafeRunSync() + actual shouldEqual NonEmptyList.of(expected) - val fetchedByTag = files.fetch(FileId(newFileId, newTag, projectRefOrg2)).accepted - fetchedByTag shouldEqual expected + val fetched = files.fetch(FileId(newFileUuid.toString, projectRefOrg2)).accepted + fetched shouldEqual expected + } } "reject if the source file doesn't exist" in { - val destination = CopyFileDestination(projectRefOrg2, None, None, None, None) - files.copyTo(fileIdIri(nxv + "other"), destination).rejectedWith[FileNotFound] + val destination = CopyFileDestination(projectRefOrg2, None, None) + val source = CopyFileSource(projectRef, NonEmptyList.of(fileIdIri(nxv + "other"))) + files.copyFiles(source, destination).rejectedWith[FileNotFound] } "reject if the destination storage doesn't exist" in { - val destination = CopyFileDestination(projectRefOrg2, None, Some(storage), None, None) - files.copyTo(fileId("file1"), destination).rejected shouldEqual + val destination = CopyFileDestination(projectRefOrg2, Some(storage), None) + val source = CopyFileSource(projectRef, NonEmptyList.of(fileId("file1"))) + files.copyFiles(source, destination).rejected shouldEqual WrappedStorageRejection(StorageNotFound(storageIri, projectRefOrg2)) } "reject if copying between different storage types" in { val expectedError = DifferentStorageType(remoteIdIri, StorageType.RemoteDiskStorage, StorageType.DiskStorage) - val destination = CopyFileDestination(projectRefOrg2, None, Some(remoteId), None, None) - files.copyTo(FileId("file1", projectRef), destination).rejected shouldEqual + val destination = CopyFileDestination(projectRefOrg2, Some(remoteId), None) + val source = CopyFileSource(projectRef, NonEmptyList.of(FileId("file1", projectRef))) + files.copyFiles(source, destination).rejected shouldEqual WrappedStorageRejection(expectedError) } + + val smallDiskCapacity = 9 + val smallDiskMaxSize = 5 + + "reject if total size of source files exceed remaining available space on the destination storage" in { + givenAFileWithSize(5) { fileId1 => + givenAFileWithSize(5) { fileId2 => + val smallDiskPayload = + diskFieldsJson deepMerge json"""{"capacity": $smallDiskCapacity, "maxFileSize": $smallDiskMaxSize, "volume": "$path"}""" + storages.create(smallDiskId, projectRefOrg2, smallDiskPayload).accepted + + val source = CopyFileSource(projectRef, NonEmptyList.of(fileId1, fileId2)) + val destination = CopyFileDestination(projectRefOrg2, Some(smallDiskId), None) + val expectedError = FileTooLarge(smallDiskMaxSize.toLong, Some(smallDiskCapacity.toLong)) + + files.copyFiles(source, destination).rejected shouldEqual expectedError + } + } + } + + "reject if any of the files exceed max file size of the destination storage" in { + givenAFileWithSize(1) { fileId1 => + givenAFileWithSize(smallDiskMaxSize + 1) { fileId2 => + val source = CopyFileSource(projectRef, NonEmptyList.of(fileId1, fileId2)) + val destination = CopyFileDestination(projectRefOrg2, Some(smallDiskId), None) + val expectedError = FileTooLarge(smallDiskMaxSize.toLong, Some(smallDiskCapacity.toLong)) + + files.copyFiles(source, destination).rejected shouldEqual expectedError + } + } + } } "deleting a tag" should { @@ -675,21 +749,22 @@ class FilesSpec(docker: RemoteStorageDocker) } - def givenAFile(assertion: FileId => Assertion): Assertion = { + def givenAFile(assertion: FileId => Assertion): Assertion = givenAFileWithSize(1)(assertion) + + def givenAFileWithSize(size: Int)(assertion: FileId => Assertion): Assertion = { val filename = genString() val id = fileId(filename) - files.create(id, Some(diskId), randomEntity(filename, 1), None).accepted + files.create(id, Some(diskId), randomEntity(filename, size), None).accepted files.fetch(id).accepted assertion(id) } - def givenADeprecatedFile(assertion: FileId => Assertion): Assertion = { + def givenADeprecatedFile(assertion: FileId => Assertion): Assertion = givenAFile { id => files.deprecate(id, 1).accepted files.fetch(id).accepted.deprecated shouldEqual true assertion(id) } - } def assertRemainsDeprecated(id: FileId): Assertion = files.fetch(id).accepted.deprecated shouldEqual true diff --git a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/Main.scala b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/Main.scala index ee3c07ed0b..d714b961f1 100644 --- a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/Main.scala +++ b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/Main.scala @@ -6,12 +6,13 @@ import akka.http.scaladsl.Http import akka.http.scaladsl.server.Route import akka.util.Timeout import cats.effect.{ExitCode, IO, IOApp} +import ch.epfl.bluebrain.nexus.delta.kernel.utils.CopyFiles import ch.epfl.bluebrain.nexus.storage.Storages.DiskStorage import ch.epfl.bluebrain.nexus.storage.attributes.{AttributesCache, ContentTypeDetector} import ch.epfl.bluebrain.nexus.storage.auth.AuthorizationMethod import ch.epfl.bluebrain.nexus.storage.config.AppConfig._ import ch.epfl.bluebrain.nexus.storage.config.{AppConfig, Settings} -import ch.epfl.bluebrain.nexus.storage.files.{CopyFiles, ValidateFile} +import ch.epfl.bluebrain.nexus.storage.files.ValidateFile import ch.epfl.bluebrain.nexus.storage.routes.Routes import com.typesafe.config.{Config, ConfigFactory} import kamon.Kamon diff --git a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/StorageError.scala b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/StorageError.scala index e04ec5ddce..1243232e15 100644 --- a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/StorageError.scala +++ b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/StorageError.scala @@ -2,10 +2,11 @@ package ch.epfl.bluebrain.nexus.storage import akka.http.scaladsl.model.StatusCodes import akka.http.scaladsl.model.Uri.Path -import ch.epfl.bluebrain.nexus.storage.files.CopyBetween +import ch.epfl.bluebrain.nexus.delta.kernel.utils.CopyBetween import ch.epfl.bluebrain.nexus.storage.routes.StatusFrom import io.circe.generic.extras.Configuration import io.circe.generic.extras.semiauto.deriveConfiguredEncoder +import io.circe.generic.semiauto.deriveEncoder import io.circe.{Encoder, Json} import scala.annotation.nowarn @@ -96,6 +97,8 @@ object StorageError { */ final case class OperationTimedOut(override val msg: String) extends StorageError(msg) + implicit val enc: Encoder[CopyBetween] = deriveEncoder + final case class CopyOperationFailed(failingCopy: CopyBetween) extends StorageError( s"Copy operation failed from source ${failingCopy.source} to destination ${failingCopy.destination}." diff --git a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/Storages.scala b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/Storages.scala index e0713211ed..6aef585398 100644 --- a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/Storages.scala +++ b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/Storages.scala @@ -6,6 +6,7 @@ import akka.stream.alpakka.file.scaladsl.Directory import akka.stream.scaladsl.{FileIO, Keep} import cats.data.{EitherT, NonEmptyList} import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.kernel.utils.{CopyBetween, CopyFiles} import ch.epfl.bluebrain.nexus.storage.File._ import ch.epfl.bluebrain.nexus.storage.Rejection.PathNotFound import ch.epfl.bluebrain.nexus.storage.StorageError.{InternalError, PermissionsFixingFailed} @@ -15,11 +16,12 @@ import ch.epfl.bluebrain.nexus.storage.Storages.{BucketExistence, PathExistence} import ch.epfl.bluebrain.nexus.storage.attributes.AttributesComputation._ import ch.epfl.bluebrain.nexus.storage.attributes.{AttributesCache, ContentTypeDetector} import ch.epfl.bluebrain.nexus.storage.config.AppConfig.{DigestConfig, StorageConfig} -import ch.epfl.bluebrain.nexus.storage.files.{CopyFileOutput, CopyFiles, ValidateFile} +import ch.epfl.bluebrain.nexus.storage.files.{CopyFileOutput, ValidateFile} import ch.epfl.bluebrain.nexus.storage.routes.CopyFile import java.nio.file.StandardCopyOption._ import java.nio.file.{Files, Path} +import fs2.io.file.{Path => Fs2Path} import java.security.MessageDigest import scala.concurrent.{ExecutionContext, Future} import scala.sys.process._ @@ -267,8 +269,10 @@ object Storages { files: NonEmptyList[CopyFile] )(implicit bucketEv: BucketExists, pathEv: PathDoesNotExist): IO[RejOr[NonEmptyList[CopyFileOutput]]] = (for { - validated <- files.traverse(f => EitherT(validateFile.forCopyWithinProtectedDir(name, f.source, f.destination))) - _ <- EitherT.right[Rejection](copyFiles.copyValidated(validated)) + validated <- files.traverse(f => EitherT(validateFile.forCopyWithinProtectedDir(name, f.source, f.destination))) + copyBetween = + validated.map(v => CopyBetween(Fs2Path.fromNioPath(v.absSourcePath), Fs2Path.fromNioPath(v.absDestPath))) + _ <- EitherT.right[Rejection](copyFiles.copyAll(copyBetween)) } yield files.zip(validated).map { case (raw, valid) => CopyFileOutput(raw.source, raw.destination, valid.absSourcePath, valid.absDestPath) }).value diff --git a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/files/CopyBetween.scala b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/files/CopyBetween.scala deleted file mode 100644 index 38aa68ca04..0000000000 --- a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/files/CopyBetween.scala +++ /dev/null @@ -1,12 +0,0 @@ -package ch.epfl.bluebrain.nexus.storage.files - -import fs2.io.file.Path -import io.circe.Encoder -import ch.epfl.bluebrain.nexus.storage._ -import io.circe.generic.semiauto.deriveEncoder - -final case class CopyBetween(source: Path, destination: Path) - -object CopyBetween { - implicit val enc: Encoder[CopyBetween] = deriveEncoder -} diff --git a/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/DiskStorageSpec.scala b/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/DiskStorageSpec.scala index 21046d6bd9..1d1e340a1f 100644 --- a/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/DiskStorageSpec.scala +++ b/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/DiskStorageSpec.scala @@ -10,6 +10,7 @@ import akka.util.ByteString import cats.data.NonEmptyList import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.kernel.http.MediaTypeDetectorConfig +import ch.epfl.bluebrain.nexus.delta.kernel.utils.CopyFiles import ch.epfl.bluebrain.nexus.storage.File.{Digest, FileAttributes} import ch.epfl.bluebrain.nexus.storage.Rejection.{PathAlreadyExists, PathNotFound} import ch.epfl.bluebrain.nexus.storage.StorageError.{PathInvalid, PermissionsFixingFailed} @@ -18,7 +19,7 @@ import ch.epfl.bluebrain.nexus.storage.Storages.DiskStorage import ch.epfl.bluebrain.nexus.storage.Storages.PathExistence.{PathDoesNotExist, PathExists} import ch.epfl.bluebrain.nexus.storage.attributes.{AttributesCache, ContentTypeDetector} import ch.epfl.bluebrain.nexus.storage.config.AppConfig.{DigestConfig, StorageConfig} -import ch.epfl.bluebrain.nexus.storage.files.{CopyFileOutput, CopyFiles, ValidateFile} +import ch.epfl.bluebrain.nexus.storage.files.{CopyFileOutput, ValidateFile} import ch.epfl.bluebrain.nexus.storage.routes.CopyFile import ch.epfl.bluebrain.nexus.storage.utils.Randomness import ch.epfl.bluebrain.nexus.testkit.scalatest.ce.CatsEffectSpec From 5c327f1be96ace9943553f22a1dd7f130bfdf0fc Mon Sep 17 00:00:00 2001 From: dantb Date: Tue, 5 Dec 2023 16:09:49 +0100 Subject: [PATCH 04/18] Return bulk response with context, test route --- .../nexus/delta/wiring/DeltaModule.scala | 4 +- .../delta/plugins/storage/files/Files.scala | 32 -------- .../plugins/storage/files/model/File.scala | 19 ++--- .../storage/files/routes/CopyFileSource.scala | 2 + .../storage/files/routes/FilesRoutes.scala | 20 +++-- .../remote/RemoteDiskStorageCopyFile.scala | 4 +- .../files/file-bulk-copy-response.json | 9 +++ .../files/routes/FilesRoutesSpec.scala | 80 +++++++++---------- .../nexus/delta/rdf/Vocabulary.scala | 1 + .../resources/contexts/bulk-operation.json | 10 +++ .../sdk/jsonld/BulkOperationResults.scala | 27 +++++++ 11 files changed, 112 insertions(+), 96 deletions(-) create mode 100644 delta/plugins/storage/src/test/resources/files/file-bulk-copy-response.json create mode 100644 delta/sdk/src/main/resources/contexts/bulk-operation.json create mode 100644 delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/jsonld/BulkOperationResults.scala diff --git a/delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/wiring/DeltaModule.scala b/delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/wiring/DeltaModule.scala index 15e5b9b204..ae3af97702 100644 --- a/delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/wiring/DeltaModule.scala +++ b/delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/wiring/DeltaModule.scala @@ -79,6 +79,7 @@ class DeltaModule(appCfg: AppConfig, config: Config)(implicit classLoader: Class make[RemoteContextResolution].named("aggregate").fromEffect { (otherCtxResolutions: Set[RemoteContextResolution]) => for { + bulkOpCtx <- ContextValue.fromFile("contexts/bulk-operation.json") errorCtx <- ContextValue.fromFile("contexts/error.json") metadataCtx <- ContextValue.fromFile("contexts/metadata.json") searchCtx <- ContextValue.fromFile("contexts/search.json") @@ -96,7 +97,8 @@ class DeltaModule(appCfg: AppConfig, config: Config)(implicit classLoader: Class contexts.remoteContexts -> remoteContextsCtx, contexts.tags -> tagsCtx, contexts.version -> versionCtx, - contexts.validation -> validationCtx + contexts.validation -> validationCtx, + contexts.bulkOperation -> bulkOpCtx ) .merge(otherCtxResolutions.toSeq: _*) } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala index 2d0893033a..1d587de1ae 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala @@ -245,38 +245,6 @@ final class Files( destinationDesc <- FileDescription(file.attributes.filename, file.attributes.mediaType) } yield CopyFileDetails(destinationDesc, file.attributes) -// /** -// * Create a file from a source file potentially in a different organization -// * -// * @param sourceId -// * File lookup id for the source file -// * @param dest -// * Project, storage and file details for the file we're creating -// */ -// def copyTo( -// sourceId: FileId, -// dest: CopyFileDestination -// )(implicit c: Caller): IO[FileResource] = { -// for { -// file <- fetchSourceFile(sourceId) -// (pc, destStorageRef, destStorage) <- fetchDestinationStorage(dest) -// _ <- validateStorageTypeForCopy(file.storageType, destStorage) -// space <- fetchStorageAvailableSpace(destStorage) -// _ <- IO.raiseUnless(space.exists(_ < file.attributes.bytes))( -// FileTooLarge(destStorage.storageValue.maxFileSize, space) -// ) -// iri <- generateId(pc) -// destinationDesc <- FileDescription(file.attributes.filename, file.attributes.mediaType) -// attributes <- CopyFile(destStorage, remoteDiskStorageClient).apply(file.attributes, destinationDesc).adaptError { -// case r: CopyFileRejection => CopyRejection(file.id, file.storage.iri, destStorage.id, r) -// } -// res <- eval(CreateFile(iri, dest.project, destStorageRef, destStorage.tpe, attributes, c.subject, dest.tag)) -// } yield res -// }.span("copyFile") - -// private def doCopy(files: NonEmptyList[CopyFileDetails]): IO[NonEmptyList[FileAttributes]] = -// - private def fetchSourceFile(id: FileId)(implicit c: Caller) = for { file <- fetch(id) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/File.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/File.scala index e33bda375c..b4a01157a4 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/File.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/File.scala @@ -48,15 +48,16 @@ object File { final case class Metadata(tags: List[UserTag]) - implicit def fileEncoder(implicit config: StorageTypeConfig): Encoder[File] = { file => - implicit val storageType: StorageType = file.storageType - val storageJson = Json.obj( - keywords.id -> file.storage.iri.asJson, - keywords.tpe -> storageType.iri.asJson, - "_rev" -> file.storage.rev.asJson - ) - file.attributes.asJson.mapObject(_.add("_storage", storageJson)) - } + implicit def fileEncoder(implicit config: StorageTypeConfig): Encoder.AsObject[File] = + Encoder.encodeJsonObject.contramapObject { file => + implicit val storageType: StorageType = file.storageType + val storageJson = Json.obj( + keywords.id -> file.storage.iri.asJson, + keywords.tpe -> storageType.iri.asJson, + "_rev" -> file.storage.rev.asJson + ) + file.attributes.asJsonObject.add("_storage", storageJson) + } implicit def fileJsonLdEncoder(implicit config: StorageTypeConfig): JsonLdEncoder[File] = JsonLdEncoder.computeFromCirce(_.id, Files.context) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/CopyFileSource.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/CopyFileSource.scala index afe5ee50bc..65cf8d8d2b 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/CopyFileSource.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/CopyFileSource.scala @@ -25,6 +25,8 @@ object CopyFileSource { case (None, Some(rev)) => Right(FileId(sourceFile, rev, proj)) case (None, None) => Right(FileId(sourceFile, proj)) case (Some(_), Some(_)) => + // TODO any decoding failures will return a 415 which isn't accurate most of the time. It should + // probably be a bad request instead. Left( DecodingFailure("Tag and revision cannot be simultaneously present for source file lookup", Nil) ) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutes.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutes.scala index 4545f4ea9e..44d9ffee15 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutes.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutes.scala @@ -12,9 +12,9 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{CopyFileDestination, File, FileId, FileRejection} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.permissions.{read => Read, write => Write} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.FilesRoutes._ -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{schemas, FileResource, Files} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{contexts, schemas, FileResource, Files} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StoragesConfig.StorageTypeConfig -import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.RemoteContextResolution +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.{ContextValue, RemoteContextResolution} import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.encoder.JsonLdEncoder import ch.epfl.bluebrain.nexus.delta.rdf.utils.JsonKeyOrdering import ch.epfl.bluebrain.nexus.delta.sdk._ @@ -27,6 +27,7 @@ import ch.epfl.bluebrain.nexus.delta.sdk.fusion.FusionConfig import ch.epfl.bluebrain.nexus.delta.sdk.identities.Identities import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller import ch.epfl.bluebrain.nexus.delta.sdk.implicits._ +import ch.epfl.bluebrain.nexus.delta.sdk.jsonld.BulkOperationResults import ch.epfl.bluebrain.nexus.delta.sdk.model.routes.Tag import ch.epfl.bluebrain.nexus.delta.sdk.model.{BaseUri, IdSegment} import ch.epfl.bluebrain.nexus.delta.sourcing.model.Tag.UserTag @@ -99,7 +100,9 @@ final class FilesRoutes( }, // Bulk create files by copying from another project entity(as[CopyFileSource]) { c: CopyFileSource => - val copyTo = CopyFileDestination(projectRef, storage, tag) + val copyTo = CopyFileDestination(projectRef, storage, tag) + implicit val bulkOpJsonLdEnc: JsonLdEncoder[BulkOperationResults[FileResource]] = + BulkOperationResults.searchResultsJsonLdEncoder(ContextValue(contexts.files)) emit(Created, copyFile(mode, c, copyTo)) }, // Create a file without id segment @@ -249,12 +252,13 @@ final class FilesRoutes( private def copyFile(mode: IndexingMode, c: CopyFileSource, copyTo: CopyFileDestination)(implicit caller: Caller - ): IO[Either[FileRejection, NonEmptyList[FileResource]]] = + ): IO[Either[FileRejection, BulkOperationResults[FileResource]]] = (for { - _ <- EitherT.right(aclCheck.authorizeForOr(c.project, Read)(AuthorizationFailed(c.project.project, Read))) - result <- EitherT(files.copyFiles(c, copyTo).attemptNarrow[FileRejection]) - _ <- EitherT.right[FileRejection](result.traverse(index(copyTo.project, _, mode))) - } yield result).value + _ <- EitherT.right(aclCheck.authorizeForOr(c.project, Read)(AuthorizationFailed(c.project.project, Read))) + result <- EitherT(files.copyFiles(c, copyTo).attemptNarrow[FileRejection]) + bulkResults = BulkOperationResults(result.toList) + _ <- EitherT.right[FileRejection](result.traverse(index(copyTo.project, _, mode))) + } yield bulkResults).value def fetch(id: FileId)(implicit caller: Caller): Route = (headerValueByType(Accept) & varyAcceptHeaders) { diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageCopyFile.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageCopyFile.scala index 4d6ddc7ec8..7d0c03c44f 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageCopyFile.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageCopyFile.scala @@ -15,14 +15,14 @@ class RemoteDiskStorageCopyFile( ) extends CopyFile { override def apply(copyDetails: NonEmptyList[CopyFileDetails]): IO[NonEmptyList[FileAttributes]] = { - val thing = copyDetails.map { cd => + val paths = copyDetails.map { cd => val destinationPath = Uri.Path(intermediateFolders(storage.project, cd.destinationDesc.uuid, cd.destinationDesc.filename)) val sourcePath = cd.sourceAttributes.location (sourcePath, destinationPath) } - client.copyFile(storage.value.folder, thing)(storage.value.endpoint).map { destPaths => + client.copyFile(storage.value.folder, paths)(storage.value.endpoint).map { destPaths => copyDetails.zip(destPaths).map { case (cd, destinationPath) => FileAttributes( uuid = cd.destinationDesc.uuid, diff --git a/delta/plugins/storage/src/test/resources/files/file-bulk-copy-response.json b/delta/plugins/storage/src/test/resources/files/file-bulk-copy-response.json new file mode 100644 index 0000000000..8781053488 --- /dev/null +++ b/delta/plugins/storage/src/test/resources/files/file-bulk-copy-response.json @@ -0,0 +1,9 @@ +{ + "@context": [ + "https://bluebrain.github.io/nexus/contexts/bulk-operation.json", + "https://bluebrain.github.io/nexus/contexts/metadata.json", + "https://bluebrain.github.io/nexus/contexts/files.json" + ], + "_total": {{total}}, + "_results": {{results}} +} \ No newline at end of file diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala index 86a15f625f..2eafea3b76 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala @@ -43,7 +43,7 @@ import ch.epfl.bluebrain.nexus.testkit.ce.IOFromMap import ch.epfl.bluebrain.nexus.testkit.errors.files.FileErrors.{fileAlreadyExistsError, fileIsNotDeprecatedError} import ch.epfl.bluebrain.nexus.testkit.scalatest.ce.CatsIOValues import io.circe.Json -import io.circe.syntax.KeyOps +import io.circe.syntax.{EncoderOps, KeyOps} import org.scalatest._ import java.util.UUID @@ -66,13 +66,14 @@ class FilesRoutesSpec // TODO: sort out how we handle this in tests implicit override def rcr: RemoteContextResolution = RemoteContextResolution.fixedIO( - storageContexts.storages -> ContextValue.fromFile("contexts/storages.json"), - storageContexts.storagesMetadata -> ContextValue.fromFile("contexts/storages-metadata.json"), - fileContexts.files -> ContextValue.fromFile("contexts/files.json"), - Vocabulary.contexts.metadata -> ContextValue.fromFile("contexts/metadata.json"), - Vocabulary.contexts.error -> ContextValue.fromFile("contexts/error.json"), - Vocabulary.contexts.tags -> ContextValue.fromFile("contexts/tags.json"), - Vocabulary.contexts.search -> ContextValue.fromFile("contexts/search.json") + storageContexts.storages -> ContextValue.fromFile("contexts/storages.json"), + storageContexts.storagesMetadata -> ContextValue.fromFile("contexts/storages-metadata.json"), + fileContexts.files -> ContextValue.fromFile("contexts/files.json"), + Vocabulary.contexts.metadata -> ContextValue.fromFile("contexts/metadata.json"), + Vocabulary.contexts.error -> ContextValue.fromFile("contexts/error.json"), + Vocabulary.contexts.tags -> ContextValue.fromFile("contexts/tags.json"), + Vocabulary.contexts.search -> ContextValue.fromFile("contexts/search.json"), + Vocabulary.contexts.bulkOperation -> ContextValue.fromFile("contexts/bulk-operation.json") ) private val reader = User("reader", realm) @@ -366,29 +367,19 @@ class FilesRoutesSpec "copy a file" in { givenAFileInProject(projectRef.toString) { oldFileId => - val newFileId = genString() - val json = Json.obj("sourceProjectRef" := projectRef, "sourceFileId" := oldFileId) + val newFileUUId = UUID.randomUUID() + withUUIDF(newFileUUId) { + val newFileId = newFileUUId.toString + val json = + Json.obj("sourceProjectRef" := projectRef, "files" := Json.arr(Json.obj("sourceFileId" := oldFileId))) - Put(s"/v1/files/${projectRefOrg2.toString}/$newFileId", json.toEntity) ~> asWriter ~> routes ~> check { - status shouldEqual StatusCodes.Created - val expectedId = project2.base.iri / newFileId - val attr = attributes(filename = oldFileId) - response.asJson shouldEqual fileMetadata(projectRefOrg2, expectedId, attr, diskIdRev) - } - } - } - - "copy a file with generated new Id" in { - val fileCopyUUId = UUID.randomUUID() - withUUIDF(fileCopyUUId) { - givenAFileInProject(projectRef.toString) { oldFileId => - val json = Json.obj("sourceProjectRef" := projectRef, "sourceFileId" := oldFileId) - - Post(s"/v1/files/${projectRefOrg2.toString}/", json.toEntity) ~> asWriter ~> routes ~> check { + Post(s"/v1/files/${projectRefOrg2.toString}", json.toEntity) ~> asWriter ~> routes ~> check { status shouldEqual StatusCodes.Created - val expectedId = project2.base.iri / fileCopyUUId.toString - val attr = attributes(filename = oldFileId, id = fileCopyUUId) - response.asJson shouldEqual fileMetadata(projectRefOrg2, expectedId, attr, diskIdRev) + val expectedId = project2.base.iri / newFileId + val expectedAttr = attributes(filename = oldFileId, id = newFileUUId) + val expectedFile = fileMetadata(projectRefOrg2, expectedId, expectedAttr, diskIdRev) + val expected = bulkOperationResponse(1, List(expectedFile)) + response.asJson shouldEqual expected } } } @@ -397,23 +388,18 @@ class FilesRoutesSpec "reject file copy request if tag and rev are present simultaneously" in { givenAFileInProject(projectRef.toString) { oldFileId => val json = Json.obj( - "sourceProjectRef" := projectRef, - "sourceFileId" := oldFileId, - "sourceTag" := "mytag", - "sourceRev" := 3 - ) - - val requests = List( - Put(s"/v1/files/${projectRefOrg2.toString}/${genString()}", json.toEntity), - Post(s"/v1/files/${projectRefOrg2.toString}/", json.toEntity) + "sourceProjectRef" := projectRef.toString, + "files" := Json.arr( + Json.obj( + "sourceFileId" := oldFileId, + "sourceTag" := "mytag", + "sourceRev" := 3 + ) + ) ) - forAll(requests) { req => - req ~> asWriter ~> routes ~> check { - status shouldEqual StatusCodes.BadRequest - response.asJson shouldEqual - jsonContentOf("/errors/tag-and-rev-copy-error.json", "fileId" -> oldFileId) - } + Post(s"/v1/files/${projectRefOrg2.toString}", json.toEntity) ~> asWriter ~> routes ~> check { + status shouldEqual StatusCodes.UnsupportedMediaType } } } @@ -724,6 +710,9 @@ class FilesRoutesSpec test(id) } + def bulkOperationResponse(total: Int, results: List[Json]): Json = + FilesRoutesSpec.bulkOperationResponse(total, results.map(_.removeKeys("@context"))).accepted + def fileMetadata( project: ProjectRef, id: Iri, @@ -777,4 +766,7 @@ object FilesRoutesSpec { "type" -> storageType, "self" -> ResourceUris("files", project, id).accessUri ) + + def bulkOperationResponse(total: Int, results: List[Json]): IO[Json] = + loader.jsonContentOf("files/file-bulk-copy-response.json", "total" -> total, "results" -> results.asJson) } diff --git a/delta/rdf/src/main/scala/ch/epfl/bluebrain/nexus/delta/rdf/Vocabulary.scala b/delta/rdf/src/main/scala/ch/epfl/bluebrain/nexus/delta/rdf/Vocabulary.scala index 327fff5517..cfefe7b018 100644 --- a/delta/rdf/src/main/scala/ch/epfl/bluebrain/nexus/delta/rdf/Vocabulary.scala +++ b/delta/rdf/src/main/scala/ch/epfl/bluebrain/nexus/delta/rdf/Vocabulary.scala @@ -216,6 +216,7 @@ object Vocabulary { val acls = contexts + "acls.json" val aclsMetadata = contexts + "acls-metadata.json" + val bulkOperation = contexts + "bulk-operation.json" val error = contexts + "error.json" val identities = contexts + "identities.json" val metadata = contexts + "metadata.json" diff --git a/delta/sdk/src/main/resources/contexts/bulk-operation.json b/delta/sdk/src/main/resources/contexts/bulk-operation.json new file mode 100644 index 0000000000..426ed55846 --- /dev/null +++ b/delta/sdk/src/main/resources/contexts/bulk-operation.json @@ -0,0 +1,10 @@ +{ + "@context": { + "_total": "https://bluebrain.github.io/nexus/vocabulary/total", + "_results": { + "@id": "https://bluebrain.github.io/nexus/vocabulary/results", + "@container": "@list" + } + }, + "@id": "https://bluebrain.github.io/nexus/contexts/bulk-operation.json" +} \ No newline at end of file diff --git a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/jsonld/BulkOperationResults.scala b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/jsonld/BulkOperationResults.scala new file mode 100644 index 0000000000..e060200843 --- /dev/null +++ b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/jsonld/BulkOperationResults.scala @@ -0,0 +1,27 @@ +package ch.epfl.bluebrain.nexus.delta.sdk.jsonld + +import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.{contexts, nxv} +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.ContextValue +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.encoder.JsonLdEncoder +import io.circe.syntax._ +import io.circe.{Encoder, Json, JsonObject} + +final case class BulkOperationResults[A](results: Seq[A]) + +object BulkOperationResults { + + private val context = ContextValue(contexts.bulkOperation, contexts.metadata) + + implicit def encoder[A: Encoder.AsObject]: Encoder.AsObject[BulkOperationResults[A]] = + Encoder.AsObject.instance { r => + JsonObject( + nxv.total.prefix -> Json.fromInt(r.results.size), + nxv.results.prefix -> Json.fromValues(r.results.map(_.asJson)) + ) + } + + def searchResultsJsonLdEncoder[A: Encoder.AsObject]( + additionalContext: ContextValue + ): JsonLdEncoder[BulkOperationResults[A]] = + JsonLdEncoder.computeFromCirce(context.merge(additionalContext)) +} From d4a5a10a57f8d7a3084047c9312f321e3ddb3159 Mon Sep 17 00:00:00 2001 From: dantb Date: Thu, 7 Dec 2023 09:02:43 +0100 Subject: [PATCH 05/18] Fix paths when saving copied file attributes --- .../nexus/delta/kernel/utils/CopyFiles.scala | 7 +- .../delta/kernel/utils/CopyFilesSuite.scala | 4 +- .../delta/plugins/storage/files/Files.scala | 33 ++++++--- .../storage/files/model/CopyFileDetails.scala | 5 +- .../storage/files/routes/FilesRoutes.scala | 5 ++ .../storage/storages/model/Storage.scala | 2 +- .../operations/disk/DiskStorageCopyFile.scala | 20 ++++-- .../remote/RemoteDiskStorageCopyFile.scala | 50 +++++++++----- .../remote/RemoteDiskStorageLinkFile.scala | 30 ++++---- .../client/RemoteDiskStorageClient.scala | 12 ++-- .../storages/routes/StoragesRoutes.scala | 2 +- .../sdk/directives/ResponseToJsonLd.scala | 42 ++++++----- .../bluebrain/nexus/storage/Storages.scala | 23 +++++-- .../nexus/storage/files/ValidateFile.scala | 32 +++++---- .../nexus/storage/routes/CopyFile.scala | 2 +- .../storage/routes/StorageDirectives.scala | 15 ++-- .../nexus/storage/routes/StorageRoutes.scala | 16 +++-- .../nexus/storage/DiskStorageSpec.scala | 41 +++++++++-- .../storage/routes/StorageRoutesSpec.scala | 56 +++++++++++---- .../bluebrain/nexus/tests/HttpClient.scala | 4 +- .../nexus/tests/kg/CopyFileSpec.scala | 56 ++++++++++----- .../nexus/tests/kg/StorageSpec.scala | 69 ++++++++----------- 22 files changed, 340 insertions(+), 186 deletions(-) diff --git a/delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/CopyFiles.scala b/delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/CopyFiles.scala index a302cec1d4..4f6970dee9 100644 --- a/delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/CopyFiles.scala +++ b/delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/CopyFiles.scala @@ -12,19 +12,20 @@ trait CopyFiles { final case class CopyBetween(source: Path, destination: Path) -final case class CopyOperationFailed(failingCopy: CopyBetween) extends Rejection { +final case class CopyOperationFailed(failingCopy: CopyBetween, e: Throwable) extends Rejection { override def reason: String = - s"Copy operation failed from source ${failingCopy.source} to destination ${failingCopy.destination}." + s"Copy operation failed from source ${failingCopy.source} to destination ${failingCopy.destination}. Underlying error: $e" } object CopyFiles { + def mk(): CopyFiles = files => copyAll(files) def copyAll(files: NonEmptyList[CopyBetween]): IO[Unit] = Ref.of[IO, Option[CopyOperationFailed]](None).flatMap { errorRef => files .parTraverse { case c @ CopyBetween(source, dest) => - copySingle(source, dest).onError(_ => errorRef.set(Some(CopyOperationFailed(c)))) + copySingle(source, dest).onError(e => errorRef.set(Some(CopyOperationFailed(c, e)))) } .void .handleErrorWith(_ => rollbackCopiesAndRethrow(errorRef, files.map(_.destination))) diff --git a/delta/kernel/src/test/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/CopyFilesSuite.scala b/delta/kernel/src/test/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/CopyFilesSuite.scala index ba426068bc..45457cbd54 100644 --- a/delta/kernel/src/test/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/CopyFilesSuite.scala +++ b/delta/kernel/src/test/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/CopyFilesSuite.scala @@ -63,7 +63,7 @@ class CopyFilesSuite extends CatsEffectSuite { error <- CopyFiles.copyAll(files).intercept[CopyOperationFailed] _ <- List(dest1, dest3, parent(dest1), parent(dest3)).traverse(fileShouldNotExist) _ <- fileShouldExist(failingDest) - } yield assertEquals(error, CopyOperationFailed(failingCopy)) + } yield assertEquals(error.failingCopy, failingCopy) } test("rollback read-only files upon failure") { @@ -78,7 +78,7 @@ class CopyFilesSuite extends CatsEffectSuite { error <- CopyFiles.copyAll(files).intercept[CopyOperationFailed] _ <- List(dest2, parent(dest2)).traverse(fileShouldNotExist) _ <- fileShouldExist(failingDest) - } yield assertEquals(error, CopyOperationFailed(failingCopy)) + } yield assertEquals(error.failingCopy, failingCopy) } def genFilePath: Path = tempDir / genString / s"$genString.txt" diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala index 1d587de1ae..968d7c5e38 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala @@ -202,11 +202,17 @@ final class Files( dest: CopyFileDestination )(implicit c: Caller): IO[NonEmptyList[FileResource]] = { for { + _ <- logger.info(s"DTBDTB entered copyFiles with $source and $dest") (pc, destStorageRef, destStorage) <- fetchDestinationStorage(dest) + _ <- logger.info(s"DTBDTB fetched dest storage") copyDetails <- source.files.traverse(fetchCopyDetails(destStorage, _)) - _ <- validateSpaceOnStorage(destStorage, copyDetails) + _ <- logger.info(s"DTBDTB fetched file attributes") + _ <- validateSpaceOnStorage(destStorage, copyDetails.map(_.sourceAttributes.bytes)) + _ <- logger.info(s"DTBDTB validated space") destFilesAttributes <- CopyFile(destStorage, remoteDiskStorageClient).apply(copyDetails) + _ <- logger.info(s"DTBDTB did actual file copy") fileResources <- evalCreateCommands(pc, dest, destStorageRef, destStorage.tpe, destFilesAttributes) + _ <- logger.info(s"DTBDTB create commands") } yield fileResources } @@ -229,28 +235,27 @@ final class Files( } yield resource } - private def validateSpaceOnStorage(destStorage: Storage, copyDetails: NonEmptyList[CopyFileDetails]): IO[Unit] = for { + private def validateSpaceOnStorage(destStorage: Storage, sourcesBytes: NonEmptyList[Long]): IO[Unit] = for { space <- fetchStorageAvailableSpace(destStorage) - allSizes = copyDetails.map(_.sourceAttributes.bytes) maxSize = destStorage.storageValue.maxFileSize - _ <- IO.raiseWhen(allSizes.exists(_ > maxSize))(FileTooLarge(maxSize, space)) - totalSize = allSizes.toList.sum + _ <- IO.raiseWhen(sourcesBytes.exists(_ > maxSize))(FileTooLarge(maxSize, space)) + totalSize = sourcesBytes.toList.sum _ <- IO.raiseWhen(space.exists(_ < totalSize))(FileTooLarge(maxSize, space)) } yield () - private def fetchCopyDetails(destStorage: Storage, fileId: FileId)(implicit c: Caller): IO[CopyFileDetails] = + private def fetchCopyDetails(destStorage: Storage, fileId: FileId)(implicit c: Caller) = for { - file <- fetchSourceFile(fileId) - _ <- validateStorageTypeForCopy(file.storageType, destStorage) - destinationDesc <- FileDescription(file.attributes.filename, file.attributes.mediaType) - } yield CopyFileDetails(destinationDesc, file.attributes) + (file, sourceStorage) <- fetchSourceFile(fileId) + _ <- validateStorageTypeForCopy(file.storageType, destStorage) + destinationDesc <- FileDescription(file.attributes.filename, file.attributes.mediaType) + } yield CopyFileDetails(destinationDesc, file.attributes, sourceStorage) private def fetchSourceFile(id: FileId)(implicit c: Caller) = for { file <- fetch(id) sourceStorage <- storages.fetch(file.value.storage, id.project) _ <- validateAuth(id.project, sourceStorage.value.storageValue.readPermission) - } yield file.value + } yield (file.value, sourceStorage.value) private def fetchDestinationStorage( dest: CopyFileDestination @@ -438,9 +443,12 @@ final class Files( def fetchContent(id: FileId)(implicit caller: Caller): IO[FileResponse] = { for { file <- fetch(id) + _ <- logger.info(s"DTBDTB fetching file content for file $file") attributes = file.value.attributes storage <- storages.fetch(file.value.storage, id.project) + _ <- logger.info(s"DTBDTB fetched storage $storage") _ <- validateAuth(id.project, storage.value.storageValue.readPermission) + _ <- logger.info(s"DTBDTB validated auth") s = fetchFile(storage.value, attributes, file.id) mediaType = attributes.mediaType.getOrElse(`application/octet-stream`) } yield FileResponse(attributes.filename, mediaType, attributes.bytes, s.attemptNarrow[FileRejection]) @@ -449,6 +457,9 @@ final class Files( private def fetchFile(storage: Storage, attr: FileAttributes, fileId: Iri): IO[AkkaSource] = FetchFile(storage, remoteDiskStorageClient, config) .apply(attr) + .onError { e => + logger.error(e)(s"DTBDTB received error during file contents fetch for $fileId") + } .adaptError { case e: FetchFileRejection => FetchRejection(fileId, storage.id, e) } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/CopyFileDetails.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/CopyFileDetails.scala index a3b31c844e..8b8b7d72ef 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/CopyFileDetails.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/CopyFileDetails.scala @@ -1,6 +1,9 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage + final case class CopyFileDetails( destinationDesc: FileDescription, - sourceAttributes: FileAttributes + sourceAttributes: FileAttributes, + sourceStorage: Storage ) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutes.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutes.scala index 44d9ffee15..df5c6d351b 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutes.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutes.scala @@ -8,6 +8,7 @@ import akka.http.scaladsl.server._ import cats.data.{EitherT, NonEmptyList} import cats.effect.IO import cats.syntax.all._ +import ch.epfl.bluebrain.nexus.delta.kernel.Logger import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{CopyFileDestination, File, FileId, FileRejection} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.permissions.{read => Read, write => Write} @@ -67,6 +68,8 @@ final class FilesRoutes( ) extends AuthDirectives(identities, aclCheck) with CirceUnmarshalling { self => + private val logger = Logger[FilesRoutes] + import baseUri.prefixSegment import schemeDirectives._ @@ -257,7 +260,9 @@ final class FilesRoutes( _ <- EitherT.right(aclCheck.authorizeForOr(c.project, Read)(AuthorizationFailed(c.project.project, Read))) result <- EitherT(files.copyFiles(c, copyTo).attemptNarrow[FileRejection]) bulkResults = BulkOperationResults(result.toList) + _ <- EitherT.right[FileRejection](logger.info(s"Indexing and returning bulk results $bulkResults")) _ <- EitherT.right[FileRejection](result.traverse(index(copyTo.project, _, mode))) + _ <- EitherT.right[FileRejection](logger.info(s"Finished indexing")) } yield bulkResults).value def fetch(id: FileId)(implicit caller: Caller): Route = diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/Storage.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/Storage.scala index d22d6f816e..9e1ede802b 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/Storage.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/Storage.scala @@ -4,11 +4,11 @@ import akka.actor.ActorSystem import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StoragesConfig.StorageTypeConfig import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.Metadata import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageValue.{DiskStorageValue, RemoteDiskStorageValue, S3StorageValue} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.{DiskStorageCopyFile, DiskStorageFetchFile, DiskStorageSaveFile} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.RemoteDiskStorageClient import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.s3.{S3StorageFetchFile, S3StorageLinkFile, S3StorageSaveFile} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{contexts, Storages} import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.ContextValue diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageCopyFile.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageCopyFile.scala index cd1674b8c8..db1d1e33d6 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageCopyFile.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageCopyFile.scala @@ -3,6 +3,7 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk import akka.http.scaladsl.model.Uri import cats.data.NonEmptyList import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.kernel.Logger import ch.epfl.bluebrain.nexus.delta.kernel.utils.{CopyBetween, CopyFiles} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{CopyFileDetails, FileAttributes} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.DiskStorage @@ -14,12 +15,14 @@ import java.net.URI import java.nio.file.Paths class DiskStorageCopyFile(storage: DiskStorage) extends CopyFile { + private val logger = Logger[DiskStorageCopyFile] override def apply(details: NonEmptyList[CopyFileDetails]): IO[NonEmptyList[FileAttributes]] = { details .traverse { copyFile => val dest = copyFile.destinationDesc val sourcePath = Paths.get(URI.create(s"file://${copyFile.sourceAttributes.location.path}")) for { + _ <- logger.info(s"DTBDTB raw source loc is $sourcePath") (destPath, destRelativePath) <- computeLocation(storage.project, storage.value, dest.uuid, dest.filename) } yield sourcePath -> FileAttributes( uuid = dest.uuid, @@ -34,11 +37,20 @@ class DiskStorageCopyFile(storage: DiskStorage) extends CopyFile { } .flatMap { destinationAttributesBySourcePath => val paths = destinationAttributesBySourcePath.map { case (sourcePath, destAttr) => - CopyBetween(Path.fromNioPath(sourcePath), Path(destAttr.location.toString())) - } - CopyFiles.copyAll(paths).as { - destinationAttributesBySourcePath.map { case (_, destAttr) => destAttr } + val source = Path.fromNioPath(sourcePath) + val dest = Path.fromNioPath(Paths.get(URI.create(s"file://${destAttr.location.path}"))) + CopyBetween(source, dest) } + logger.info(s"DTBDTB about to do file copy for paths $paths") >> + CopyFiles + .copyAll(paths) + .onError { e => + logger.error(s"DTBDTB disk file copy failed with $e") + + } + .as { + destinationAttributesBySourcePath.map { case (_, destAttr) => destAttr } + } <* logger.info("DTBDTB completed file copy") } } } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageCopyFile.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageCopyFile.scala index 7d0c03c44f..d92b084d05 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageCopyFile.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageCopyFile.scala @@ -3,39 +3,53 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote import akka.http.scaladsl.model.Uri import cats.data.NonEmptyList import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.kernel.Logger import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{CopyFileDetails, FileAttributes} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.RemoteDiskStorage +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageValue import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.CopyFile import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.SaveFile.intermediateFolders import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.RemoteDiskStorageClient class RemoteDiskStorageCopyFile( - storage: RemoteDiskStorage, + destStorage: RemoteDiskStorage, client: RemoteDiskStorageClient ) extends CopyFile { + private val logger = Logger[RemoteDiskStorageCopyFile] + override def apply(copyDetails: NonEmptyList[CopyFileDetails]): IO[NonEmptyList[FileAttributes]] = { - val paths = copyDetails.map { cd => + val maybePaths = copyDetails.traverse { cd => val destinationPath = - Uri.Path(intermediateFolders(storage.project, cd.destinationDesc.uuid, cd.destinationDesc.filename)) - val sourcePath = cd.sourceAttributes.location - (sourcePath, destinationPath) - } + Uri.Path(intermediateFolders(destStorage.project, cd.destinationDesc.uuid, cd.destinationDesc.filename)) + val sourcePath = cd.sourceAttributes.path - client.copyFile(storage.value.folder, paths)(storage.value.endpoint).map { destPaths => - copyDetails.zip(destPaths).map { case (cd, destinationPath) => - FileAttributes( - uuid = cd.destinationDesc.uuid, - location = destinationPath, - path = destinationPath.path, - filename = cd.destinationDesc.filename, - mediaType = cd.destinationDesc.mediaType, - bytes = cd.sourceAttributes.bytes, - digest = cd.sourceAttributes.digest, - origin = cd.sourceAttributes.origin - ) + val thingy = cd.sourceStorage.storageValue match { + case remote: StorageValue.RemoteDiskStorageValue => IO(remote.folder) + case other => IO.raiseError(new Exception(s"Invalid storage type for remote copy: $other")) } + thingy.map(sourceBucket => (sourceBucket, sourcePath, destinationPath)) } + maybePaths.flatMap { paths => + logger.info(s"DTBDTB REMOTE doing copy with ${destStorage.value.folder} and $paths") >> + client.copyFile(destStorage.value.folder, paths)(destStorage.value.endpoint).flatMap { destPaths => + logger.info(s"DTBDTB REMOTE received destPaths ${destPaths}").as { + copyDetails.zip(paths).zip(destPaths).map { case ((cd, x), destinationPath) => + FileAttributes( + uuid = cd.destinationDesc.uuid, + location = destinationPath, + path = x._3, + filename = cd.destinationDesc.filename, + mediaType = cd.destinationDesc.mediaType, + bytes = cd.sourceAttributes.bytes, + digest = cd.sourceAttributes.digest, + origin = cd.sourceAttributes.origin + ) + } + } + } + } } + } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageLinkFile.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageLinkFile.scala index 087d6c8436..2c4cd47f7c 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageLinkFile.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageLinkFile.scala @@ -2,6 +2,7 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote import akka.http.scaladsl.model.Uri import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.kernel.Logger import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes.FileAttributesOrigin.Storage import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes, FileDescription} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.RemoteDiskStorage @@ -12,21 +13,24 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote. class RemoteDiskStorageLinkFile(storage: RemoteDiskStorage, client: RemoteDiskStorageClient) extends LinkFile { + private val logger = Logger[RemoteDiskStorageLinkFile] + def apply(sourcePath: Uri.Path, description: FileDescription): IO[FileAttributes] = { val destinationPath = Uri.Path(intermediateFolders(storage.project, description.uuid, description.filename)) - client.moveFile(storage.value.folder, sourcePath, destinationPath)(storage.value.endpoint).map { - case RemoteDiskStorageFileAttributes(location, bytes, digest, _) => - FileAttributes( - uuid = description.uuid, - location = location, - path = destinationPath, - filename = description.filename, - mediaType = description.mediaType, - bytes = bytes, - digest = digest, - origin = Storage - ) - } + logger.info(s"DTBDTB doing link file with ${storage.value.folder}, source $sourcePath, dest $destinationPath") >> + client.moveFile(storage.value.folder, sourcePath, destinationPath)(storage.value.endpoint).map { + case RemoteDiskStorageFileAttributes(location, bytes, digest, _) => + FileAttributes( + uuid = description.uuid, + location = location, + path = destinationPath, + filename = description.filename, + mediaType = description.mediaType, + bytes = bytes, + digest = digest, + origin = Storage + ) + } } } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/RemoteDiskStorageClient.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/RemoteDiskStorageClient.scala index 7f33a75e7c..bf7a9013a7 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/RemoteDiskStorageClient.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/RemoteDiskStorageClient.scala @@ -180,7 +180,7 @@ final class RemoteDiskStorageClient(client: HttpClient, getAuthToken: AuthTokenP /** * Moves a path from the provided ''sourceRelativePath'' to ''destRelativePath'' inside the nexus folder. * - * @param bucket + * @param destBucket * the storage bucket name * @param sourceRelativePath * the source relative path location @@ -188,13 +188,13 @@ final class RemoteDiskStorageClient(client: HttpClient, getAuthToken: AuthTokenP * the destination relative path location inside the nexus folder */ def copyFile( - bucket: Label, - files: NonEmptyList[(Uri, Path)] + destBucket: Label, + files: NonEmptyList[(Label, Path, Path)] )(implicit baseUri: BaseUri): IO[NonEmptyList[Uri]] = { getAuthToken(credentials).flatMap { authToken => - val endpoint = baseUri.endpoint / "buckets" / bucket.value / "files" - val payload = files.map { case (source, dest) => - Json.obj("source" := source.toString(), "destination" := dest.toString()) + val endpoint = baseUri.endpoint / "buckets" / destBucket.value / "files" + val payload = files.map { case (sourceBucket, source, dest) => + Json.obj("sourceBucket" := sourceBucket, "source" := source.toString(), "destination" := dest.toString()) }.asJson implicit val dec: Decoder[NonEmptyList[Uri]] = Decoder[NonEmptyList[Json]].emap { nel => diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/routes/StoragesRoutes.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/routes/StoragesRoutes.scala index 5cb04cd343..7f6cac7df6 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/routes/StoragesRoutes.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/routes/StoragesRoutes.scala @@ -60,7 +60,7 @@ final class StoragesRoutes( (baseUriPrefix(baseUri.prefix) & replaceUri("storages", schemas.storage)) { pathPrefix("storages") { extractCaller { implicit caller => - resolveProjectRef.apply { implicit ref => + resolveProjectRef.apply { ref => concat( (pathEndOrSingleSlash & operationName(s"$prefixSegment/storages/{org}/{project}")) { // Create a storage without id segment diff --git a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/directives/ResponseToJsonLd.scala b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/directives/ResponseToJsonLd.scala index 33a3823ea9..b7bb3926b6 100644 --- a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/directives/ResponseToJsonLd.scala +++ b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/directives/ResponseToJsonLd.scala @@ -10,12 +10,13 @@ import akka.http.scaladsl.server.Route import cats.effect.IO import cats.effect.unsafe.implicits._ import cats.syntax.all._ +import ch.epfl.bluebrain.nexus.delta.kernel.Logger import ch.epfl.bluebrain.nexus.delta.rdf.RdfMediaTypes._ import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.{JsonLdApi, JsonLdJavaApi} import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.RemoteContextResolution import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.encoder.JsonLdEncoder import ch.epfl.bluebrain.nexus.delta.rdf.utils.JsonKeyOrdering -import ch.epfl.bluebrain.nexus.delta.sdk.JsonLdValue +import ch.epfl.bluebrain.nexus.delta.sdk.{AkkaSource, JsonLdValue} import ch.epfl.bluebrain.nexus.delta.sdk.directives.ResponseToJsonLd.{RouteOutcome, UseLeft, UseRight} import ch.epfl.bluebrain.nexus.delta.sdk.directives.DeltaDirectives.{emit, jsonLdFormatOrReject, mediaTypes, requestMediaType, unacceptedMediaTypeRejection} import ch.epfl.bluebrain.nexus.delta.sdk.directives.Response.{Complete, Reject} @@ -114,31 +115,38 @@ object ResponseToJsonLd extends FileBytesInstances { val encodedFilename = Base64.getEncoder.encodeToString(filename.getBytes(StandardCharsets.UTF_8)) s"=?UTF-8?B?$encodedFilename?=" } - + private val logger = Logger[ResponseToJsonLd] override def apply(statusOverride: Option[StatusCode]): Route = { val flattened = io.flatMap { _.traverse { fr => - fr.content.map { - _.map { s => - fr.metadata -> s + logger.info(s"DTBDTB in file response JSON LD encoding for ${fr.metadata.filename}") >> + fr.content.map { + _.map { s => + fr.metadata -> s + } } - } } } onSuccess(flattened.unsafeToFuture()) { - case Left(complete: Complete[E]) => emit(complete) - case Left(reject: Reject[E]) => emit(reject) - case Right(Left(c)) => emit(c) - case Right(Right((metadata, content))) => - headerValueByType(Accept) { accept => - if (accept.mediaRanges.exists(_.matches(metadata.contentType.mediaType))) { - val encodedFilename = attachmentString(metadata.filename) - respondWithHeaders(RawHeader("Content-Disposition", s"""attachment; filename="$encodedFilename"""")) { - complete(statusOverride.getOrElse(OK), HttpEntity(metadata.contentType, content)) + thing: Either[Response[E], Either[Complete[JsonLdValue], (FileResponse.Metadata, AkkaSource)]] => + logger.info(s"DTBDTB result of encoding was $thing").unsafeRunSync() + thing match { + case Left(complete: Complete[E]) => emit(complete) + case Left(reject: Reject[E]) => emit(reject) + case Right(Left(c)) => emit(c) + case Right(Right((metadata, content))) => + headerValueByType(Accept) { accept => + if (accept.mediaRanges.exists(_.matches(metadata.contentType.mediaType))) { + val encodedFilename = attachmentString(metadata.filename) + respondWithHeaders( + RawHeader("Content-Disposition", s"""attachment; filename="$encodedFilename"""") + ) { + complete(statusOverride.getOrElse(OK), HttpEntity(metadata.contentType, content)) + } + } else + reject(unacceptedMediaTypeRejection(Seq(metadata.contentType.mediaType))) } - } else - reject(unacceptedMediaTypeRejection(Seq(metadata.contentType.mediaType))) } } } diff --git a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/Storages.scala b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/Storages.scala index 6aef585398..2ef96c7147 100644 --- a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/Storages.scala +++ b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/Storages.scala @@ -130,15 +130,21 @@ trait Storages[Source] { object Storages { - sealed trait BucketExistence - sealed trait PathExistence { + sealed trait BucketExistence { + def exists: Boolean + } + sealed trait PathExistence { def exists: Boolean } object BucketExistence { - final case object BucketExists extends BucketExistence - final case object BucketDoesNotExist extends BucketExistence - type BucketExists = BucketExists.type + final case object BucketExists extends BucketExistence { + val exists = true + } + final case object BucketDoesNotExist extends BucketExistence { + val exists = false + } + type BucketExists = BucketExists.type type BucketDoesNotExist = BucketDoesNotExist.type } @@ -265,11 +271,14 @@ object Storages { IO.raiseError(InternalError(s"Path '$absPath' is not a file nor a directory")) def copyFiles( - name: String, + destBucket: String, files: NonEmptyList[CopyFile] )(implicit bucketEv: BucketExists, pathEv: PathDoesNotExist): IO[RejOr[NonEmptyList[CopyFileOutput]]] = (for { - validated <- files.traverse(f => EitherT(validateFile.forCopyWithinProtectedDir(name, f.source, f.destination))) + validated <- + files.traverse(f => + EitherT(validateFile.forCopyWithinProtectedDir(f.sourceBucket, destBucket, f.source, f.destination)) + ) copyBetween = validated.map(v => CopyBetween(Fs2Path.fromNioPath(v.absSourcePath), Fs2Path.fromNioPath(v.absDestPath))) _ <- EitherT.right[Rejection](copyFiles.copyAll(copyBetween)) diff --git a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/files/ValidateFile.scala b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/files/ValidateFile.scala index ac005b0302..312a292bf1 100644 --- a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/files/ValidateFile.scala +++ b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/files/ValidateFile.scala @@ -25,7 +25,8 @@ trait ValidateFile { ): IO[RejOr[ValidatedMoveFile]] def forCopyWithinProtectedDir( - name: String, + sourceBucket: String, + destBucket: String, sourcePath: Uri.Path, destPath: Uri.Path ): IO[RejOr[ValidatedCopyFile]] @@ -33,7 +34,12 @@ trait ValidateFile { sealed abstract case class ValidatedCreateFile(absDestPath: Path) sealed abstract case class ValidatedMoveFile(name: String, absSourcePath: Path, absDestPath: Path, isDir: Boolean) -sealed abstract case class ValidatedCopyFile(name: String, absSourcePath: Path, absDestPath: Path) +sealed abstract case class ValidatedCopyFile( + sourceBucket: String, + destBucket: String, + absSourcePath: Path, + absDestPath: Path +) object ValidateFile { @@ -71,25 +77,27 @@ object ValidateFile { } override def forCopyWithinProtectedDir( - name: String, + sourceBucket: String, + destBucket: String, sourcePath: Uri.Path, destPath: Uri.Path ): IO[RejOr[ValidatedCopyFile]] = { - val bucketProtectedPath = basePath(config, name) - val absSourcePath = filePath(config, name, sourcePath) - val absDestPath = filePath(config, name, destPath) + val sourceBucketProtectedPath = basePath(config, sourceBucket) + val destBucketProtectedPath = basePath(config, destBucket) + val absSourcePath = filePath(config, sourceBucket, sourcePath) + val absDestPath = filePath(config, destBucket, destPath) - def notFound = PathNotFound(name, sourcePath) + def notFound = PathNotFound(destBucket, sourcePath) (for { _ <- rejectIf(fileExists(absSourcePath).map(!_), notFound) - _ <- rejectIf((!descendantOf(absSourcePath, bucketProtectedPath)).pure[IO], notFound) - _ <- throwIf(!descendantOf(absDestPath, bucketProtectedPath), PathInvalid(name, destPath)) - _ <- rejectIf(fileExists(absDestPath), PathAlreadyExists(name, destPath)) + _ <- rejectIf((!descendantOf(absSourcePath, sourceBucketProtectedPath)).pure[IO], notFound) + _ <- throwIf(!descendantOf(absDestPath, destBucketProtectedPath), PathInvalid(destBucket, destPath)) + _ <- rejectIf(fileExists(absDestPath), PathAlreadyExists(destBucket, destPath)) isFile <- EitherT.right[Rejection](isRegularFile(absSourcePath)) - _ <- throwIf(!isFile, PathInvalid(name, sourcePath)) - } yield new ValidatedCopyFile(name, absSourcePath, absDestPath) {}).value + _ <- throwIf(!isFile, PathInvalid(sourceBucket, sourcePath)) + } yield new ValidatedCopyFile(sourceBucket, destBucket, absSourcePath, absDestPath) {}).value } def fileExists(absSourcePath: Path): IO[Boolean] = IO.blocking(Files.exists(absSourcePath)) diff --git a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/routes/CopyFile.scala b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/routes/CopyFile.scala index b794d33486..97dc73994b 100644 --- a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/routes/CopyFile.scala +++ b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/routes/CopyFile.scala @@ -5,7 +5,7 @@ import ch.epfl.bluebrain.nexus.storage._ import io.circe.Decoder import io.circe.generic.semiauto.deriveDecoder -final case class CopyFile(source: Uri.Path, destination: Uri.Path) +final case class CopyFile(sourceBucket: String, source: Uri.Path, destination: Uri.Path) object CopyFile { implicit val dec: Decoder[CopyFile] = deriveDecoder } diff --git a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/routes/StorageDirectives.scala b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/routes/StorageDirectives.scala index 25198e131d..2014b28759 100644 --- a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/routes/StorageDirectives.scala +++ b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/routes/StorageDirectives.scala @@ -39,11 +39,11 @@ object StorageDirectives { def validatePath(name: String, path: Path): Directive0 = if (pathInvalid(path)) failWith(PathInvalid(name, path)) else pass - def validatePaths(name: String, paths: NonEmptyList[Path]): Directive0 = - paths + def validatePaths(pathsByBucket: NonEmptyList[(String, Path)]): Directive0 = + pathsByBucket .collectFirst[Directive0] { - case p if pathInvalid(p) => - failWith(PathInvalid(name, p)) + case (bucket, p) if pathInvalid(p) => + failWith(PathInvalid(bucket, p)) } .getOrElse(pass) @@ -71,6 +71,13 @@ object StorageDirectives { case _ => reject(BucketNotFound(name)) } + def bucketsExist(buckets: NonEmptyList[String])(implicit storages: Storages[_]): Directive1[BucketExists] = + buckets + .map(storages.exists) + .zip(buckets) + .collectFirst[Directive1[BucketExists]] { case (e, bucket) if !e.exists => reject(BucketNotFound(bucket)) } + .getOrElse(provide(BucketExists)) + /** * Returns the evidence that a path exists * diff --git a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/routes/StorageRoutes.scala b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/routes/StorageRoutes.scala index 8451dbbaf5..7ca6accc78 100644 --- a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/routes/StorageRoutes.scala +++ b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/routes/StorageRoutes.scala @@ -8,6 +8,7 @@ import akka.http.scaladsl.server.Route import cats.data.NonEmptyList import cats.effect.unsafe.implicits._ import cats.syntax.all._ +import ch.epfl.bluebrain.nexus.delta.kernel.Logger import ch.epfl.bluebrain.nexus.storage.File.{Digest, FileAttributes} import ch.epfl.bluebrain.nexus.storage.config.AppConfig import ch.epfl.bluebrain.nexus.storage.config.AppConfig.HttpConfig @@ -22,6 +23,8 @@ import kamon.instrumentation.akka.http.TracingDirectives.operationName class StorageRoutes()(implicit storages: Storages[AkkaSource], hc: HttpConfig) { + private val logger = Logger[StorageRoutes] + def routes: Route = // Consume buckets/{name}/ (encodeResponse & pathPrefix("buckets" / Segment)) { name => @@ -72,12 +75,17 @@ class StorageRoutes()(implicit storages: Storages[AkkaSource], hc: HttpConfig) { }, operationName(s"/${hc.prefix}/buckets/{}/files") { post { - // Copy files within protected directory + // Copy files within protected directory between potentially different buckets entity(as[CopyFilePayload]) { payload => val files = payload.files - pathsDoNotExist(name, files.map(_.destination)).apply { implicit pathNotExistEvidence => - validatePaths(name, files.map(_.source)) { - complete(storages.copyFiles(name, files).runWithStatus(Created)) + bucketsExist(files.map(_.sourceBucket)).apply { implicit bucketExistsEvidence => + pathsDoNotExist(name, files.map(_.destination)).apply { implicit pathNotExistEvidence => + validatePaths(files.map(c => c.sourceBucket -> c.source)) { + complete( + (logger.info(s"Received request to copy files: $files") >> + storages.copyFiles(name, files)).runWithStatus(Created) + ) + } } } } diff --git a/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/DiskStorageSpec.scala b/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/DiskStorageSpec.scala index 1d1e340a1f..04c21cd2c5 100644 --- a/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/DiskStorageSpec.scala +++ b/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/DiskStorageSpec.scala @@ -337,7 +337,7 @@ class DiskStorageSpec "fail when source does not exists" in new AbsoluteDirectoryCreated { val source = randomString() - val files = NonEmptyList.of(CopyFile(Uri.Path(source), Uri.Path(randomString()))) + val files = NonEmptyList.of(CopyFile(name, Uri.Path(source), Uri.Path(randomString()))) storage.copyFiles(name, files).accepted.leftValue shouldEqual PathNotFound(name, Uri.Path(source)) } @@ -346,7 +346,7 @@ class DiskStorageSpec val absoluteFile = baseRootPath.resolve(Paths.get(file)) Files.createDirectories(absoluteFile.getParent) Files.write(absoluteFile, "something".getBytes(StandardCharsets.UTF_8)) - val files = NonEmptyList.of(CopyFile(Uri.Path(file), Uri.Path(randomString()))) + val files = NonEmptyList.of(CopyFile(name, Uri.Path(file), Uri.Path(randomString()))) storage.copyFiles(name, files).accepted.leftValue shouldEqual PathNotFound(name, Uri.Path(file)) } @@ -360,7 +360,7 @@ class DiskStorageSpec val destFile = "destFile.txt" val resolvedDestFile = basePath.resolve(Paths.get(destFile)) Files.write(resolvedDestFile, "somethingelse".getBytes(StandardCharsets.UTF_8)) - val files = NonEmptyList.of(CopyFile(Uri.Path(file), Uri.Path(destFile))) + val files = NonEmptyList.of(CopyFile(name, Uri.Path(file), Uri.Path(destFile))) storage.copyFiles(name, files).accepted.leftValue shouldEqual PathAlreadyExists(name, Uri.Path(destFile)) } @@ -373,14 +373,14 @@ class DiskStorageSpec val content = "some content" Files.write(absoluteFile, content.getBytes(StandardCharsets.UTF_8)) - val files = NonEmptyList.of(CopyFile(Uri.Path(file), dest)) + val files = NonEmptyList.of(CopyFile(name, Uri.Path(file), dest)) storage.copyFiles(name, files).rejectedWith[StorageError] shouldEqual PathInvalid(name, dest) Files.exists(absoluteFile) shouldEqual true } - "pass on file specified for absolute/relative path" in { + "pass on file in same bucket specified for absolute/relative path" in { forAll(List(true, false)) { useRelativePath => new AbsoluteDirectoryCreated { val sourceFile = sConfig.protectedDirectory.toString + "/my !file.txt" @@ -394,7 +394,7 @@ class DiskStorageSpec val absoluteDestFile = basePath.resolve(Paths.get(destPath)) val sourcePathToUse = if (useRelativePath) sourceFile else absoluteSourceFile.toString - val files = NonEmptyList.of(CopyFile(Uri.Path(sourcePathToUse), Uri.Path(destPath))) + val files = NonEmptyList.of(CopyFile(name, Uri.Path(sourcePathToUse), Uri.Path(destPath))) val expectedOutput = CopyFileOutput(Uri.Path(sourcePathToUse), Uri.Path(destPath), absoluteSourceFile, absoluteDestFile) @@ -407,6 +407,35 @@ class DiskStorageSpec } } + "pass on file in different bucket specified for absolute/relative path" in { + forAll(List(true, false)) { useRelativePath => + val dest: AbsoluteDirectoryCreated = new AbsoluteDirectoryCreated {} + val source: AbsoluteDirectoryCreated = new AbsoluteDirectoryCreated {} + val sourceFile = sConfig.protectedDirectory.toString + "/my !file.txt" + val absoluteSourceFile = source.basePath.resolve(Paths.get(sourceFile)) + Files.createDirectories(absoluteSourceFile.getParent) + + val content = "some content" + Files.write(absoluteSourceFile, content.getBytes(StandardCharsets.UTF_8)) + + val destPath = "some/other path.txt" + val absoluteDestFile = dest.basePath.resolve(Paths.get(destPath)) + + val sourcePathToUse = if (useRelativePath) sourceFile else absoluteSourceFile.toString + val files = NonEmptyList.of(CopyFile(source.name, Uri.Path(sourcePathToUse), Uri.Path(destPath))) + val expectedOutput = + CopyFileOutput(Uri.Path(sourcePathToUse), Uri.Path(destPath), absoluteSourceFile, absoluteDestFile) + + Files.exists(absoluteSourceFile) shouldEqual true + + storage.copyFiles(dest.name, files).accepted shouldEqual Right(NonEmptyList.of(expectedOutput)) + + Files.exists(absoluteSourceFile) shouldEqual true + Files.exists(absoluteDestFile) shouldEqual true + Files.readString(absoluteDestFile, StandardCharsets.UTF_8) shouldEqual content + } + } + } "fetching" should { diff --git a/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/routes/StorageRoutesSpec.scala b/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/routes/StorageRoutesSpec.scala index b3e8a1f1ed..89cb7a6f66 100644 --- a/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/routes/StorageRoutesSpec.scala +++ b/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/routes/StorageRoutesSpec.scala @@ -290,10 +290,11 @@ class StorageRoutesSpec "copying a file" should { - "fail when bucket does not exist" in new Ctx { + "fail when destination bucket does not exist" in new Ctx { storages.exists(name) shouldReturn BucketDoesNotExist - val json = Json.arr(Json.obj("source" := "source/dir", "destination" := "/path/to/myfile.txt")) + val json = + Json.arr(Json.obj("sourceBucket" := name, "source" := "source/dir", "destination" := "/path/to/myfile.txt")) Post(s"/v1/buckets/$name/files", json) ~> route ~> check { status shouldEqual NotFound @@ -308,6 +309,29 @@ class StorageRoutesSpec } } + "fail when a source bucket does not exist" in new Ctx { + val sourceBucket = randomString() + storages.exists(name) shouldReturn BucketExists + storages.exists(sourceBucket) shouldReturn BucketDoesNotExist + + val json = Json.arr( + Json.obj("sourceBucket" := sourceBucket, "source" := "source/dir", "destination" := "/path/to/myfile.txt") + ) + + Post(s"/v1/buckets/$name/files", json) ~> route ~> check { + status shouldEqual NotFound + responseAs[Json] shouldEqual jsonContentOf( + "/error.json", + Map( + quote("{type}") -> "BucketNotFound", + quote("{reason}") -> s"The provided bucket '$sourceBucket' does not exist." + ) + ) + storages.exists(name) wasCalled once + storages.exists(sourceBucket) wasCalled once + } + } + "fail if an empty array is passed" in new Ctx { storages.exists(name) shouldReturn BucketExists @@ -322,11 +346,11 @@ class StorageRoutesSpec val source = "source/dir" val dest = "dest/dir" storages.pathExists(name, Uri.Path(dest)) shouldReturn PathDoesNotExist - val input = NonEmptyList.of(CopyFile(Uri.Path(source), Uri.Path(dest))) + val input = NonEmptyList.of(CopyFile(name, Uri.Path(source), Uri.Path(dest))) storages.copyFiles(name, input)(BucketExists, PathDoesNotExist) shouldReturn IO.raiseError(InternalError("something went wrong")) - val json = Json.arr(Json.obj("source" := source, "destination" := dest)) + val json = Json.arr(Json.obj("sourceBucket" := name, "source" := source, "destination" := dest)) Post(s"/v1/buckets/$name/files", json) ~> route ~> check { status shouldEqual InternalServerError @@ -347,7 +371,7 @@ class StorageRoutesSpec val dest = "dest/dir" storages.pathExists(name, Uri.Path(dest)) shouldReturn PathDoesNotExist - val json = Json.arr(Json.obj("source" := source, "destination" := dest)) + val json = Json.arr(Json.obj("sourceBucket" := name, "source" := source, "destination" := dest)) Post(s"/v1/buckets/$name/files", json) ~> route ~> check { status shouldEqual BadRequest @@ -364,19 +388,21 @@ class StorageRoutesSpec } "pass" in new Ctx { + val sourceBucket = randomString() storages.exists(name) shouldReturn BucketExists - val source = "source/dir" - val dest = "dest/dir" - val output = + storages.exists(sourceBucket) shouldReturn BucketExists + val source = "source/dir" + val dest = "dest/dir" + val output = CopyFileOutput(Uri.Path(source), Uri.Path(dest), Paths.get(s"/rootdir/$source"), Paths.get(s"/rootdir/$dest")) storages.pathExists(name, Uri.Path(dest)) shouldReturn PathDoesNotExist - storages.copyFiles(name, NonEmptyList.of(CopyFile(Uri.Path(source), Uri.Path(dest))))( + storages.copyFiles(name, NonEmptyList.of(CopyFile(sourceBucket, Uri.Path(source), Uri.Path(dest))))( BucketExists, PathDoesNotExist ) shouldReturn IO.pure(Right(NonEmptyList.of(output))) - val json = Json.arr(Json.obj("source" := source, "destination" := dest)) + val json = Json.arr(Json.obj("sourceBucket" := sourceBucket, "source" := source, "destination" := dest)) val response = Json.arr( Json.obj( "sourcePath" := source, @@ -390,7 +416,7 @@ class StorageRoutesSpec status shouldEqual Created responseAs[Json] shouldEqual response - storages.copyFiles(name, NonEmptyList.of(CopyFile(Uri.Path(source), Uri.Path(dest))))( + storages.copyFiles(name, NonEmptyList.of(CopyFile(sourceBucket, Uri.Path(source), Uri.Path(dest))))( BucketExists, PathDoesNotExist ) wasCalled once @@ -406,8 +432,8 @@ class StorageRoutesSpec storages.pathExists(name, Uri.Path(existingDest)) shouldReturn PathExists val json = Json.arr( - Json.obj("source" := source, "destination" := dest), - Json.obj("source" := source, "destination" := existingDest) + Json.obj("sourceBucket" := name, "source" := source, "destination" := dest), + Json.obj("sourceBucket" := name, "source" := source, "destination" := existingDest) ) Post(s"/v1/buckets/$name/files", json) ~> route ~> check { @@ -423,8 +449,8 @@ class StorageRoutesSpec storages.pathExists(name, Uri.Path(dest)) shouldReturn PathDoesNotExist val json = Json.arr( - Json.obj("source" := source, "destination" := dest), - Json.obj("source" := invalidSource, "destination" := dest) + Json.obj("sourceBucket" := name, "source" := source, "destination" := dest), + Json.obj("sourceBucket" := name, "source" := invalidSource, "destination" := dest) ) Post(s"/v1/buckets/$name/files", json) ~> route ~> check { diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/HttpClient.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/HttpClient.scala index bfe91b256e..bc82e9dd76 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/HttpClient.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/HttpClient.scala @@ -66,10 +66,10 @@ class HttpClient private (baseUrl: Uri, httpExt: HttpExt)(implicit )(implicit um: FromEntityUnmarshaller[A]): IO[Assertion] = requestAssert(PUT, url, Some(body), identity, extraHeaders)(assertResponse) - def putAndReturn[A](url: String, body: Json, identity: Identity, extraHeaders: Seq[HttpHeader] = jsonHeaders)( + def postAndReturn[A](url: String, body: Json, identity: Identity, extraHeaders: Seq[HttpHeader] = jsonHeaders)( assertResponse: (A, HttpResponse) => (A, Assertion) )(implicit um: FromEntityUnmarshaller[A]): IO[A] = - requestAssertAndReturn(PUT, url, Some(body), identity, extraHeaders)(assertResponse).map(_._1) + requestAssertAndReturn(POST, url, Some(body), identity, extraHeaders)(assertResponse).map(_._1) def putIO[A](url: String, body: IO[Json], identity: Identity, extraHeaders: Seq[HttpHeader] = jsonHeaders)( assertResponse: (A, HttpResponse) => Assertion diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/CopyFileSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/CopyFileSpec.scala index 19b4c60858..f0a310c90b 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/CopyFileSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/CopyFileSpec.scala @@ -1,19 +1,21 @@ package ch.epfl.bluebrain.nexus.tests.kg -import akka.http.scaladsl.model._ +import akka.http.scaladsl.model.StatusCodes import akka.util.ByteString import cats.effect.IO +import cats.implicits.toTraverseOps import ch.epfl.bluebrain.nexus.delta.kernel.utils.UrlUtils import ch.epfl.bluebrain.nexus.tests.HttpClient._ import ch.epfl.bluebrain.nexus.tests.Identity.storages.Coyote -import ch.epfl.bluebrain.nexus.tests.Optics -import io.circe.Json import io.circe.syntax.KeyOps +import io.circe.{Decoder, DecodingFailure, Json, JsonObject} import org.scalatest.Assertion +import scala.concurrent.duration.DurationInt + trait CopyFileSpec { self: StorageSpec => - "Copying a json file to a different organization" should { + "Copying multiple files to a different organization" should { def givenAProjectWithStorage(test: String => IO[Assertion]): IO[Assertion] = { val (proj, org) = (genId(), genId()) @@ -23,29 +25,45 @@ trait CopyFileSpec { self: StorageSpec => test(projRef) } + final case class Response(ids: List[String]) + implicit val dec: Decoder[Response] = Decoder.instance { cur => + cur + .get[List[JsonObject]]("_results") + .flatMap(_.traverse(_.apply("@id").flatMap(_.asString).toRight(DecodingFailure("Missing id", Nil)))) + .map(Response) + } + "succeed" in { givenAProjectWithStorage { destProjRef => - val sourceFileId = "attachment.json" - val destFileId = "attachment2.json" - val destFilename = genId() + val sourceFiles = List(emptyTextFile, updatedJsonFileWithContentType, textFileWithContentType) + val sourcePayloads = sourceFiles.map(f => Json.obj("sourceFileId" := f.fileId)) val payload = Json.obj( - "destinationFilename" := destFilename, - "sourceProjectRef" := self.projectRef, - "sourceFileId" := sourceFileId + "sourceProjectRef" := self.projectRef, + "files" := sourcePayloads ) - val uri = s"/files/$destProjRef/$destFileId?storage=nxv:$storageId" + val uri = s"/files/$destProjRef?storage=nxv:$storageId" for { - json <- deltaClient.putAndReturn[Json](uri, payload, Coyote) { (json, response) => - (json, expectCreated(json, response)) - } - returnedId = Optics.`@id`.getOption(json).getOrElse(fail("could not find @id of created resource")) - assertion <- - deltaClient.get[ByteString](s"/files/$destProjRef/${UrlUtils.encode(returnedId)}", Coyote, acceptAll) { - expectDownload(destFilename, ContentTypes.`application/json`, updatedJsonFileContent) + response <- deltaClient.postAndReturn[Response](uri, payload, Coyote) { (json, response) => + (json, expectCreated(json, response)) + } + _ <- IO.sleep(5.seconds) + _ <- response.ids.traverse { id => + deltaClient.get[Json](s"/files/$destProjRef/${UrlUtils.encode(id)}", Coyote) { (json, response) => + println(s"Received json for id $id: $json") + response.status shouldEqual StatusCodes.OK + } + } + assertions <- + response.ids.zip(sourceFiles).traverse { case (destId, Input(_, filename, contentType, contents)) => + println(s"Fetching file $destId") + deltaClient + .get[ByteString](s"/files/$destProjRef/${UrlUtils.encode(destId)}", Coyote, acceptAll) { + expectDownload(filename, contentType, contents) + } } - } yield assertion + } yield assertions.head } } } diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/StorageSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/StorageSpec.scala index 42e3104295..a1316e500e 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/StorageSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/StorageSpec.scala @@ -19,6 +19,7 @@ import org.scalatest.Assertion import java.util.Base64 +final case class Input(fileId: String, filename: String, ct: ContentType, contents: String) abstract class StorageSpec extends BaseIntegrationSpec { val storageConfig: StorageConfig = load[StorageConfig](ConfigFactory.load(), "storage") @@ -48,9 +49,18 @@ abstract class StorageSpec extends BaseIntegrationSpec { private[tests] val fileSelfPrefix = fileSelf(projectRef, attachmentPrefix) + val emptyFileContent = "" val jsonFileContent = """{ "initial": ["is", "a", "test", "file"] }""" val updatedJsonFileContent = """{ "updated": ["is", "a", "test", "file"] }""" + val emptyTextFile = Input("empty", "empty", ContentTypes.`text/plain(UTF-8)`, emptyFileContent) + val jsonFileNoContentType = Input("attachment.json", "attachment.json", ContentTypes.NoContentType, jsonFileContent) + val updatedJsonFileWithContentType = + jsonFileNoContentType.copy(contents = updatedJsonFileContent, ct = ContentTypes.`application/json`) + val textFileNoContentType = Input("attachment2", "attachment2", ContentTypes.NoContentType, "text file") + val textFileWithContentType = + Input("attachment3", "attachment2", ContentTypes.`application/octet-stream`, "text file") + override def beforeAll(): Unit = { super.beforeAll() createProjects(Coyote, orgId, projId).accepted @@ -73,16 +83,8 @@ abstract class StorageSpec extends BaseIntegrationSpec { "An empty file" should { - val emptyFileContent = "" - "be successfully uploaded" in { - deltaClient.uploadFile[Json]( - s"/files/$projectRef/empty?storage=nxv:$storageId", - emptyFileContent, - ContentTypes.`text/plain(UTF-8)`, - "empty", - Coyote - ) { expectCreated } + uploadFile(emptyTextFile, None)(expectCreated) } "be downloaded" in { @@ -95,15 +97,7 @@ abstract class StorageSpec extends BaseIntegrationSpec { "A json file" should { "be uploaded" in { - deltaClient.uploadFile[Json]( - s"/files/$projectRef/attachment.json?storage=nxv:$storageId", - jsonFileContent, - ContentTypes.NoContentType, - "attachment.json", - Coyote - ) { - expectCreated - } + uploadFile(jsonFileNoContentType, None)(expectCreated) } "be downloaded" in { @@ -119,15 +113,7 @@ abstract class StorageSpec extends BaseIntegrationSpec { } "be updated" in { - deltaClient.uploadFile[Json]( - s"/files/$projectRef/attachment.json?storage=nxv:$storageId&rev=1", - updatedJsonFileContent, - ContentTypes.`application/json`, - "attachment.json", - Coyote - ) { - expectOk - } + uploadFile(updatedJsonFileWithContentType, Some(1))(expectOk) } "download the updated file" in { @@ -184,23 +170,17 @@ abstract class StorageSpec extends BaseIntegrationSpec { "A file without extension" should { - val textFileContent = "text file" - "be uploaded" in { - deltaClient.uploadFile[Json]( - s"/files/$projectRef/attachment2?storage=nxv:$storageId", - textFileContent, - ContentTypes.NoContentType, - "attachment2", - Coyote - ) { - expectCreated - } + uploadFile(textFileNoContentType, None)(expectCreated) } "be downloaded" in { deltaClient.get[ByteString](s"/files/$projectRef/attachment:attachment2", Coyote, acceptAll) { - expectDownload("attachment2", ContentTypes.`application/octet-stream`, textFileContent) + expectDownload( + textFileNoContentType.filename, + ContentTypes.`application/octet-stream`, + textFileNoContentType.contents + ) } } } @@ -419,6 +399,17 @@ abstract class StorageSpec extends BaseIntegrationSpec { } } + private def uploadFile(fileInput: Input, rev: Option[Int]): ((Json, HttpResponse) => Assertion) => IO[Assertion] = { + val revString = rev.map(r => s"&rev=$r").getOrElse("") + deltaClient.uploadFile[Json]( + s"/files/$projectRef/${fileInput.fileId}?storage=nxv:$storageId$revString", + fileInput.contents, + fileInput.ct, + fileInput.filename, + Coyote + ) + } + private def attachmentString(filename: String): String = { val encodedFilename = new String(Base64.getEncoder.encode(filename.getBytes(Charsets.UTF_8))) s"=?UTF-8?B?$encodedFilename?=" From 2a37f7799b249d64b8315e9fe89b43cc1f2316d1 Mon Sep 17 00:00:00 2001 From: dantb Date: Thu, 7 Dec 2023 15:09:43 +0100 Subject: [PATCH 06/18] Refactoring --- ...es.scala => TransactionalFileCopier.scala} | 16 ++- ...ala => TransactionalFileCopierSuite.scala} | 16 +-- .../plugins/storage/StoragePluginModule.scala | 5 +- .../delta/plugins/storage/files/Files.scala | 80 ++++++------- .../storage/storages/StoragesStatistics.scala | 13 ++- .../storage/storages/model/Storage.scala | 9 +- .../{CopyFile.scala => CopyFiles.scala} | 13 ++- .../operations/disk/DiskStorageCopyFile.scala | 56 ---------- .../disk/DiskStorageCopyFiles.scala | 54 +++++++++ .../disk/DiskStorageFetchFile.scala | 12 +- .../storages/operations/disk/package.scala | 17 +++ ...scala => RemoteDiskStorageCopyFiles.scala} | 8 +- .../plugins/storage/files/FilesSpec.scala | 25 +---- .../files/routes/FilesRoutesSpec.scala | 5 +- .../epfl/bluebrain/nexus/storage/Main.scala | 8 +- .../bluebrain/nexus/storage/Storages.scala | 4 +- .../nexus/storage/DiskStorageSpec.scala | 4 +- .../nexus/tests/kg/CopyFileSpec.scala | 105 ++++++++++-------- .../nexus/tests/kg/DiskStorageSpec.scala | 14 +-- .../nexus/tests/kg/RemoteStorageSpec.scala | 18 +-- .../nexus/tests/kg/S3StorageSpec.scala | 18 +-- .../nexus/tests/kg/StorageSpec.scala | 16 ++- 22 files changed, 274 insertions(+), 242 deletions(-) rename delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/{CopyFiles.scala => TransactionalFileCopier.scala} (78%) rename delta/kernel/src/test/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/{CopyFilesSuite.scala => TransactionalFileCopierSuite.scala} (91%) rename delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/{CopyFile.scala => CopyFiles.scala} (75%) delete mode 100644 delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageCopyFile.scala create mode 100644 delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageCopyFiles.scala create mode 100644 delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/package.scala rename delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/{RemoteDiskStorageCopyFile.scala => RemoteDiskStorageCopyFiles.scala} (94%) diff --git a/delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/CopyFiles.scala b/delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/TransactionalFileCopier.scala similarity index 78% rename from delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/CopyFiles.scala rename to delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/TransactionalFileCopier.scala index 4f6970dee9..49dba83df7 100644 --- a/delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/CopyFiles.scala +++ b/delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/TransactionalFileCopier.scala @@ -3,10 +3,11 @@ package ch.epfl.bluebrain.nexus.delta.kernel.utils import cats.data.NonEmptyList import cats.effect.{IO, Ref} import cats.implicits._ +import ch.epfl.bluebrain.nexus.delta.kernel.Logger import ch.epfl.bluebrain.nexus.delta.kernel.error.Rejection import fs2.io.file.{CopyFlag, CopyFlags, Files, Path} -trait CopyFiles { +trait TransactionalFileCopier { def copyAll(files: NonEmptyList[CopyBetween]): IO[Unit] } @@ -17,18 +18,23 @@ final case class CopyOperationFailed(failingCopy: CopyBetween, e: Throwable) ext s"Copy operation failed from source ${failingCopy.source} to destination ${failingCopy.destination}. Underlying error: $e" } -object CopyFiles { +object TransactionalFileCopier { - def mk(): CopyFiles = files => copyAll(files) + private val logger = Logger[TransactionalFileCopier] - def copyAll(files: NonEmptyList[CopyBetween]): IO[Unit] = + def mk(): TransactionalFileCopier = files => copyAll(files) + + private def copyAll(files: NonEmptyList[CopyBetween]): IO[Unit] = Ref.of[IO, Option[CopyOperationFailed]](None).flatMap { errorRef => files .parTraverse { case c @ CopyBetween(source, dest) => copySingle(source, dest).onError(e => errorRef.set(Some(CopyOperationFailed(c, e)))) } .void - .handleErrorWith(_ => rollbackCopiesAndRethrow(errorRef, files.map(_.destination))) + .handleErrorWith { e => + logger.error(e)("Transactional files copy failed, deleting created files") >> + rollbackCopiesAndRethrow(errorRef, files.map(_.destination)) + } } def parent(p: Path): Path = Path.fromNioPath(p.toNioPath.getParent) diff --git a/delta/kernel/src/test/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/CopyFilesSuite.scala b/delta/kernel/src/test/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/TransactionalFileCopierSuite.scala similarity index 91% rename from delta/kernel/src/test/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/CopyFilesSuite.scala rename to delta/kernel/src/test/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/TransactionalFileCopierSuite.scala index 45457cbd54..3625ed5bc9 100644 --- a/delta/kernel/src/test/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/CopyFilesSuite.scala +++ b/delta/kernel/src/test/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/TransactionalFileCopierSuite.scala @@ -3,7 +3,7 @@ package ch.epfl.bluebrain.nexus.delta.kernel.utils import cats.data.NonEmptyList import cats.effect.IO import cats.syntax.all._ -import ch.epfl.bluebrain.nexus.delta.kernel.utils.CopyFiles.parent +import ch.epfl.bluebrain.nexus.delta.kernel.utils.TransactionalFileCopier.parent import fs2.io.file.PosixPermission._ import fs2.io.file._ import munit.CatsEffectSuite @@ -11,19 +11,21 @@ import munit.catseffect.IOFixture import java.util.UUID -class CopyFilesSuite extends CatsEffectSuite { +class TransactionalFileCopierSuite extends CatsEffectSuite { val myFixture: IOFixture[Path] = ResourceSuiteLocalFixture("create-temp-dir-fixture", Files[IO].tempDirectory) override def munitFixtures: List[IOFixture[Path]] = List(myFixture) lazy val tempDir: Path = myFixture() + val copier: TransactionalFileCopier = TransactionalFileCopier.mk() + test("successfully copy contents of multiple files") { for { (source1, source1Contents) <- givenAFileExists (source2, source2Contents) <- givenAFileExists (dest1, dest2, dest3) = (genFilePath, genFilePath, genFilePath) files = NonEmptyList.of(CopyBetween(source1, dest1), CopyBetween(source2, dest2), CopyBetween(source1, dest3)) - _ <- CopyFiles.copyAll(files) + _ <- copier.copyAll(files) _ <- fileShouldExistWithContents(source1Contents, dest1) _ <- fileShouldExistWithContents(source1Contents, dest3) _ <- fileShouldExistWithContents(source2Contents, dest2) @@ -36,7 +38,7 @@ class CopyFilesSuite extends CatsEffectSuite { sourceAttr <- Files[IO].getBasicFileAttributes(source) dest = genFilePath files = NonEmptyList.of(CopyBetween(source, dest)) - _ <- CopyFiles.copyAll(files) + _ <- copier.copyAll(files) _ <- fileShouldExistWithContentsAndAttributes(dest, contents, sourceAttr) } yield () } @@ -48,7 +50,7 @@ class CopyFilesSuite extends CatsEffectSuite { (source, _) <- givenAFileWithPermissions(sourcePermissions) dest = genFilePath files = NonEmptyList.of(CopyBetween(source, dest)) - _ <- CopyFiles.copyAll(files) + _ <- copier.copyAll(files) _ <- fileShouldExistWithPermissions(dest, sourcePermissions) } yield () } @@ -60,7 +62,7 @@ class CopyFilesSuite extends CatsEffectSuite { (dest1, dest3) = (genFilePath, genFilePath) failingCopy = CopyBetween(source, failingDest) files = NonEmptyList.of(CopyBetween(source, dest1), failingCopy, CopyBetween(source, dest3)) - error <- CopyFiles.copyAll(files).intercept[CopyOperationFailed] + error <- copier.copyAll(files).intercept[CopyOperationFailed] _ <- List(dest1, dest3, parent(dest1), parent(dest3)).traverse(fileShouldNotExist) _ <- fileShouldExist(failingDest) } yield assertEquals(error.failingCopy, failingCopy) @@ -75,7 +77,7 @@ class CopyFilesSuite extends CatsEffectSuite { dest2 = genFilePath failingCopy = CopyBetween(source, failingDest) files = NonEmptyList.of(CopyBetween(source, dest2), failingCopy) - error <- CopyFiles.copyAll(files).intercept[CopyOperationFailed] + error <- copier.copyAll(files).intercept[CopyOperationFailed] _ <- List(dest2, parent(dest2)).traverse(fileShouldNotExist) _ <- fileShouldExist(failingDest) } yield assertEquals(error.failingCopy, failingCopy) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala index 19efc3c516..c7add71b57 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala @@ -3,7 +3,7 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage import akka.actor import akka.actor.typed.ActorSystem import cats.effect.{Clock, IO} -import ch.epfl.bluebrain.nexus.delta.kernel.utils.{ClasspathResourceLoader, UUIDF} +import ch.epfl.bluebrain.nexus.delta.kernel.utils.{ClasspathResourceLoader, TransactionalFileCopier, UUIDF} import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.client.ElasticSearchClient import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.config.ElasticSearchViewsConfig import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.Files @@ -174,7 +174,8 @@ class StoragePluginModule(priority: Int) extends ModuleDef { storageTypeConfig, cfg.files, remoteDiskStorageClient, - clock + clock, + TransactionalFileCopier.mk() )( uuidF, as diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala index 968d7c5e38..9a9c89202b 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala @@ -9,7 +9,7 @@ import cats.effect.{Clock, IO} import cats.syntax.all._ import ch.epfl.bluebrain.nexus.delta.kernel.cache.LocalCache import ch.epfl.bluebrain.nexus.delta.kernel.kamon.KamonMetricComponent -import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF +import ch.epfl.bluebrain.nexus.delta.kernel.utils.{TransactionalFileCopier, UUIDF} import ch.epfl.bluebrain.nexus.delta.kernel.{Logger, RetryStrategy} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.Files._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest.{ComputedDigest, NotComputedDigest} @@ -63,7 +63,8 @@ final class Files( storages: Storages, storagesStatistics: StoragesStatistics, remoteDiskStorageClient: RemoteDiskStorageClient, - config: StorageTypeConfig + config: StorageTypeConfig, + copier: TransactionalFileCopier )(implicit uuidF: UUIDF, system: ClassicActorSystem @@ -202,17 +203,11 @@ final class Files( dest: CopyFileDestination )(implicit c: Caller): IO[NonEmptyList[FileResource]] = { for { - _ <- logger.info(s"DTBDTB entered copyFiles with $source and $dest") (pc, destStorageRef, destStorage) <- fetchDestinationStorage(dest) - _ <- logger.info(s"DTBDTB fetched dest storage") copyDetails <- source.files.traverse(fetchCopyDetails(destStorage, _)) - _ <- logger.info(s"DTBDTB fetched file attributes") _ <- validateSpaceOnStorage(destStorage, copyDetails.map(_.sourceAttributes.bytes)) - _ <- logger.info(s"DTBDTB validated space") - destFilesAttributes <- CopyFile(destStorage, remoteDiskStorageClient).apply(copyDetails) - _ <- logger.info(s"DTBDTB did actual file copy") + destFilesAttributes <- CopyFiles(destStorage, remoteDiskStorageClient, copier).apply(copyDetails) fileResources <- evalCreateCommands(pc, dest, destStorageRef, destStorage.tpe, destFilesAttributes) - _ <- logger.info(s"DTBDTB create commands") } yield fileResources } @@ -236,7 +231,7 @@ final class Files( } private def validateSpaceOnStorage(destStorage: Storage, sourcesBytes: NonEmptyList[Long]): IO[Unit] = for { - space <- fetchStorageAvailableSpace(destStorage) + space <- storagesStatistics.getStorageAvailableSpace(destStorage) maxSize = destStorage.storageValue.maxFileSize _ <- IO.raiseWhen(sourcesBytes.exists(_ > maxSize))(FileTooLarge(maxSize, space)) totalSize = sourcesBytes.toList.sum @@ -443,12 +438,9 @@ final class Files( def fetchContent(id: FileId)(implicit caller: Caller): IO[FileResponse] = { for { file <- fetch(id) - _ <- logger.info(s"DTBDTB fetching file content for file $file") attributes = file.value.attributes storage <- storages.fetch(file.value.storage, id.project) - _ <- logger.info(s"DTBDTB fetched storage $storage") _ <- validateAuth(id.project, storage.value.storageValue.readPermission) - _ <- logger.info(s"DTBDTB validated auth") s = fetchFile(storage.value, attributes, file.id) mediaType = attributes.mediaType.getOrElse(`application/octet-stream`) } yield FileResponse(attributes.filename, mediaType, attributes.bytes, s.attemptNarrow[FileRejection]) @@ -457,9 +449,6 @@ final class Files( private def fetchFile(storage: Storage, attr: FileAttributes, fileId: Iri): IO[AkkaSource] = FetchFile(storage, remoteDiskStorageClient, config) .apply(attr) - .onError { e => - logger.error(e)(s"DTBDTB received error during file contents fetch for $fileId") - } .adaptError { case e: FetchFileRejection => FetchRejection(fileId, storage.id, e) } @@ -472,21 +461,7 @@ final class Files( * @param project * the project where the storage belongs */ - def fetch(id: FileId): IO[FileResource] = { - for { - (iri, _) <- id.expandIri(fetchContext.onRead) - state <- fetchState(id, iri) - } yield state.toResource - }.span("fetchFile") - - private def fetchState(id: FileId, iri: Iri): IO[FileState] = { - val notFound = FileNotFound(iri, id.project) - id.id match { - case Latest(_) => log.stateOr(id.project, iri, notFound) - case Revision(_, rev) => log.stateOr(id.project, iri, rev, notFound, RevisionNotFound) - case Tag(_, tag) => log.stateOr(id.project, iri, tag, notFound, TagNotFound(tag)) - } - } + def fetch(id: FileId): IO[FileResource] = Files.fetch(fetchContext, log)(id).span("fetchFile") private def createLink( iri: Iri, @@ -548,7 +523,7 @@ final class Files( private def extractFormData(iri: Iri, storage: Storage, entity: HttpEntity): IO[(FileDescription, BodyPartEntity)] = for { - storageAvailableSpace <- fetchStorageAvailableSpace(storage) + storageAvailableSpace <- storagesStatistics.getStorageAvailableSpace(storage) (description, source) <- formDataExtractor(iri, entity, storage.storageValue.maxFileSize, storageAvailableSpace) } yield (description, source) @@ -557,17 +532,7 @@ final class Files( .apply(description, source) .adaptError { case e: SaveFileRejection => SaveRejection(iri, storage.id, e) } - private def fetchStorageAvailableSpace(storage: Storage): IO[Option[Long]] = - storage.storageValue.capacity.fold(IO.none[Long]) { capacity => - storagesStatistics - .get(storage.id, storage.project) - .redeem( - _ => Some(capacity), - stat => Some(capacity - stat.spaceUsed) - ) - } - - private def expandStorageIri(segment: IdSegment, pc: ProjectContext): IO[Iri] = + private def expandStorageIri(segment: IdSegment, pc: ProjectContext): IO[Iri] = Storages.expandIri(segment, pc).adaptError { case s: StorageRejection => WrappedStorageRejection(s) } @@ -860,7 +825,8 @@ object Files { storageTypeConfig: StorageTypeConfig, config: FilesConfig, remoteDiskStorageClient: RemoteDiskStorageClient, - clock: Clock[IO] + clock: Clock[IO], + copier: TransactionalFileCopier )(implicit uuidF: UUIDF, as: ActorSystem[Nothing] @@ -874,7 +840,8 @@ object Files { storages, storagesStatistics, remoteDiskStorageClient, - storageTypeConfig + storageTypeConfig, + copier ) } @@ -893,4 +860,27 @@ object Files { .void } + /** + * Fetch the last version of a file + * + * @param id + * the identifier that will be expanded to the Iri of the file with its optional rev/tag + * @param project + * the project where the storage belongs + */ + def fetch(fetchContext: FetchContext[FileRejection], log: FilesLog)(id: FileId): IO[FileResource] = + for { + (iri, _) <- id.expandIri(fetchContext.onRead) + state <- fetchState(log)(id, iri) + } yield state.toResource + + private def fetchState(log: FilesLog)(id: FileId, iri: Iri): IO[FileState] = { + val notFound = FileNotFound(iri, id.project) + id.id match { + case Latest(_) => log.stateOr(id.project, iri, notFound) + case Revision(_, rev) => log.stateOr(id.project, iri, rev, notFound, RevisionNotFound) + case Tag(_, tag) => log.stateOr(id.project, iri, tag, notFound, TagNotFound(tag)) + } + } + } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StoragesStatistics.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StoragesStatistics.scala index e047e2c92e..84f11822dd 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StoragesStatistics.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StoragesStatistics.scala @@ -4,7 +4,7 @@ import akka.http.scaladsl.model.Uri.Query import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.EventMetricsProjection.eventMetricsIndex import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.client.ElasticSearchClient -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageStatEntry +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{Storage, StorageStatEntry} import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.sdk.model.IdSegment import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef @@ -18,6 +18,17 @@ trait StoragesStatistics { */ def get(idSegment: IdSegment, project: ProjectRef): IO[StorageStatEntry] + /** + * Retrieve remaining space on a storage if it has a capacity. + */ + final def getStorageAvailableSpace(storage: Storage): IO[Option[Long]] = + storage.storageValue.capacity.fold(IO.none[Long]) { capacity => + get(storage.id, storage.project) + .redeem( + _ => Some(capacity), + stat => Some(capacity - stat.spaceUsed) + ) + } } object StoragesStatistics { diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/Storage.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/Storage.scala index 9e1ede802b..110d08db5f 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/Storage.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/Storage.scala @@ -1,11 +1,12 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model import akka.actor.ActorSystem +import ch.epfl.bluebrain.nexus.delta.kernel.utils.TransactionalFileCopier import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StoragesConfig.StorageTypeConfig import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.Metadata import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageValue.{DiskStorageValue, RemoteDiskStorageValue, S3StorageValue} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations._ -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.{DiskStorageCopyFile, DiskStorageFetchFile, DiskStorageSaveFile} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.{DiskStorageCopyFiles, DiskStorageFetchFile, DiskStorageSaveFile} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.RemoteDiskStorageClient import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.s3.{S3StorageFetchFile, S3StorageLinkFile, S3StorageSaveFile} @@ -89,7 +90,7 @@ object Storage { def saveFile(implicit as: ActorSystem): SaveFile = new DiskStorageSaveFile(this) - def copyFile: CopyFile = new DiskStorageCopyFile(this) + def copyFiles(copier: TransactionalFileCopier): CopyFiles = new DiskStorageCopyFiles(this, copier) } /** @@ -139,8 +140,8 @@ object Storage { def linkFile(client: RemoteDiskStorageClient): LinkFile = new RemoteDiskStorageLinkFile(this, client) - def copyFile(client: RemoteDiskStorageClient): CopyFile = - new RemoteDiskStorageCopyFile(this, client) + def copyFiles(client: RemoteDiskStorageClient): CopyFiles = + new RemoteDiskStorageCopyFiles(this, client) def fetchComputedAttributes(client: RemoteDiskStorageClient): FetchAttributes = new RemoteStorageFetchAttributes(value, client) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/CopyFile.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/CopyFiles.scala similarity index 75% rename from delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/CopyFile.scala rename to delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/CopyFiles.scala index 64c81cdf90..3e2382b2a6 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/CopyFile.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/CopyFiles.scala @@ -2,25 +2,26 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations import cats.data.NonEmptyList import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.kernel.utils.TransactionalFileCopier import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{CopyFileDetails, FileAttributes} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{Storage, StorageType} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.CopyFileRejection import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.RemoteDiskStorageClient -trait CopyFile { +trait CopyFiles { def apply(copyDetails: NonEmptyList[CopyFileDetails]): IO[NonEmptyList[FileAttributes]] } -object CopyFile { +object CopyFiles { - def apply(storage: Storage, client: RemoteDiskStorageClient): CopyFile = + def apply(storage: Storage, client: RemoteDiskStorageClient, copier: TransactionalFileCopier): CopyFiles = storage match { - case storage: Storage.DiskStorage => storage.copyFile + case storage: Storage.DiskStorage => storage.copyFiles(copier) case storage: Storage.S3Storage => unsupported(storage.tpe) - case storage: Storage.RemoteDiskStorage => storage.copyFile(client) + case storage: Storage.RemoteDiskStorage => storage.copyFiles(client) } - private def unsupported(storageType: StorageType): CopyFile = + private def unsupported(storageType: StorageType): CopyFiles = _ => IO.raiseError(CopyFileRejection.UnsupportedOperation(storageType)) } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageCopyFile.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageCopyFile.scala deleted file mode 100644 index db1d1e33d6..0000000000 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageCopyFile.scala +++ /dev/null @@ -1,56 +0,0 @@ -package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk - -import akka.http.scaladsl.model.Uri -import cats.data.NonEmptyList -import cats.effect.IO -import ch.epfl.bluebrain.nexus.delta.kernel.Logger -import ch.epfl.bluebrain.nexus.delta.kernel.utils.{CopyBetween, CopyFiles} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{CopyFileDetails, FileAttributes} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.DiskStorage -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.CopyFile -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.DiskStorageSaveFile.computeLocation -import fs2.io.file.Path - -import java.net.URI -import java.nio.file.Paths - -class DiskStorageCopyFile(storage: DiskStorage) extends CopyFile { - private val logger = Logger[DiskStorageCopyFile] - override def apply(details: NonEmptyList[CopyFileDetails]): IO[NonEmptyList[FileAttributes]] = { - details - .traverse { copyFile => - val dest = copyFile.destinationDesc - val sourcePath = Paths.get(URI.create(s"file://${copyFile.sourceAttributes.location.path}")) - for { - _ <- logger.info(s"DTBDTB raw source loc is $sourcePath") - (destPath, destRelativePath) <- computeLocation(storage.project, storage.value, dest.uuid, dest.filename) - } yield sourcePath -> FileAttributes( - uuid = dest.uuid, - location = Uri(destPath.toUri.toString), - path = Uri.Path(destRelativePath.toString), - filename = dest.filename, - mediaType = copyFile.sourceAttributes.mediaType, - bytes = copyFile.sourceAttributes.bytes, - digest = copyFile.sourceAttributes.digest, - origin = copyFile.sourceAttributes.origin - ) - } - .flatMap { destinationAttributesBySourcePath => - val paths = destinationAttributesBySourcePath.map { case (sourcePath, destAttr) => - val source = Path.fromNioPath(sourcePath) - val dest = Path.fromNioPath(Paths.get(URI.create(s"file://${destAttr.location.path}"))) - CopyBetween(source, dest) - } - logger.info(s"DTBDTB about to do file copy for paths $paths") >> - CopyFiles - .copyAll(paths) - .onError { e => - logger.error(s"DTBDTB disk file copy failed with $e") - - } - .as { - destinationAttributesBySourcePath.map { case (_, destAttr) => destAttr } - } <* logger.info("DTBDTB completed file copy") - } - } -} diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageCopyFiles.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageCopyFiles.scala new file mode 100644 index 0000000000..4397e18d32 --- /dev/null +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageCopyFiles.scala @@ -0,0 +1,54 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk + +import akka.http.scaladsl.model.Uri +import cats.data.NonEmptyList +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.kernel.utils.{CopyBetween, TransactionalFileCopier} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{CopyFileDetails, FileAttributes} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.DiskStorage +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.CopyFiles +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.DiskStorageSaveFile.computeLocation +import fs2.io.file.Path + +import java.nio.file + +class DiskStorageCopyFiles(storage: DiskStorage, copier: TransactionalFileCopier) extends CopyFiles { + + override def apply(details: NonEmptyList[CopyFileDetails]): IO[NonEmptyList[FileAttributes]] = + details + .traverse(mkCopyDetailsAndDestAttributes) + .flatMap { copyDetailsAndDestAttributes => + val copyDetails = copyDetailsAndDestAttributes.map(_._1) + val destAttrs = copyDetailsAndDestAttributes.map(_._2) + copier.copyAll(copyDetails).as(destAttrs) + } + + private def mkCopyDetailsAndDestAttributes(copyFile: CopyFileDetails) = + for { + sourcePath <- absoluteDiskPathFromAttributes(copyFile.sourceAttributes) + (destPath, destRelativePath) <- computeLocation( + storage.project, + storage.value, + copyFile.destinationDesc.uuid, + copyFile.destinationDesc.filename + ) + destAttr = mkDestAttributes(copyFile, destPath, destRelativePath) + copyDetails <- absoluteDiskPathFromAttributes(destAttr).map { dest => + CopyBetween(Path.fromNioPath(sourcePath), Path.fromNioPath(dest)) + } + } yield (copyDetails, destAttr) + + private def mkDestAttributes(copyFile: CopyFileDetails, destPath: file.Path, destRelativePath: file.Path) = { + val dest = copyFile.destinationDesc + FileAttributes( + uuid = dest.uuid, + location = Uri(destPath.toUri.toString), + path = Uri.Path(destRelativePath.toString), + filename = dest.filename, + mediaType = copyFile.sourceAttributes.mediaType, + bytes = copyFile.sourceAttributes.bytes, + digest = copyFile.sourceAttributes.digest, + origin = copyFile.sourceAttributes.origin + ) + } +} diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageFetchFile.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageFetchFile.scala index 5f3cd155a9..8f80e4b238 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageFetchFile.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageFetchFile.scala @@ -9,20 +9,16 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.Storage import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.FetchFileRejection.UnexpectedLocationFormat import ch.epfl.bluebrain.nexus.delta.sdk.AkkaSource -import java.net.URI -import java.nio.file.Paths -import scala.util.{Failure, Success, Try} - object DiskStorageFetchFile extends FetchFile { override def apply(attributes: FileAttributes): IO[AkkaSource] = apply(attributes.location.path) override def apply(path: Uri.Path): IO[AkkaSource] = - Try(Paths.get(URI.create(s"file://$path"))) match { - case Failure(err) => IO.raiseError(UnexpectedLocationFormat(s"file://$path", err.getMessage)) - case Success(path) => + absoluteDiskPath(path).redeemWith( + e => IO.raiseError(UnexpectedLocationFormat(s"file://$path", e.getMessage)), + path => IO.raiseWhen(!path.toFile.exists())(FetchFileRejection.FileNotFound(path.toString)) .map(_ => FileIO.fromPath(path)) - } + ) } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/package.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/package.scala new file mode 100644 index 0000000000..996e861fc4 --- /dev/null +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/package.scala @@ -0,0 +1,17 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations + +import akka.http.scaladsl.model.Uri.Path +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes + +import java.net.URI +import java.nio.file +import java.nio.file.Paths + +package object disk { + + def absoluteDiskPathFromAttributes(attr: FileAttributes): IO[file.Path] = absoluteDiskPath(attr.location.path) + + def absoluteDiskPath(relative: Path): IO[file.Path] = IO(Paths.get(URI.create(s"file://$relative"))) + +} diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageCopyFile.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageCopyFiles.scala similarity index 94% rename from delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageCopyFile.scala rename to delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageCopyFiles.scala index d92b084d05..63a72bd0a3 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageCopyFile.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageCopyFiles.scala @@ -7,16 +7,16 @@ import ch.epfl.bluebrain.nexus.delta.kernel.Logger import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{CopyFileDetails, FileAttributes} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.RemoteDiskStorage import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageValue -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.CopyFile +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.CopyFiles import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.SaveFile.intermediateFolders import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.RemoteDiskStorageClient -class RemoteDiskStorageCopyFile( +class RemoteDiskStorageCopyFiles( destStorage: RemoteDiskStorage, client: RemoteDiskStorageClient -) extends CopyFile { +) extends CopyFiles { - private val logger = Logger[RemoteDiskStorageCopyFile] + private val logger = Logger[RemoteDiskStorageCopyFiles] override def apply(copyDetails: NonEmptyList[CopyFileDetails]): IO[NonEmptyList[FileAttributes]] = { val maybePaths = copyDetails.traverse { cd => diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala index 19f4272c04..436007f0f1 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala @@ -9,6 +9,7 @@ import cats.data.NonEmptyList import cats.effect.IO import cats.effect.unsafe.implicits.global import ch.epfl.bluebrain.nexus.delta.kernel.http.MediaTypeDetectorConfig +import ch.epfl.bluebrain.nexus.delta.kernel.utils.TransactionalFileCopier import ch.epfl.bluebrain.nexus.delta.plugins.storage.RemoteContextResolutionFixture import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest.NotComputedDigest import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes.FileAttributesOrigin.Storage @@ -138,7 +139,8 @@ class FilesSpec(docker: RemoteStorageDocker) cfg, FilesConfig(eventLogConfig, MediaTypeDetectorConfig.Empty), remoteDiskStorageClient, - clock + clock, + TransactionalFileCopier.mk() ) def fileId(file: String): FileId = FileId(file, projectRef) @@ -452,7 +454,7 @@ class FilesSpec(docker: RemoteStorageDocker) "succeed from disk storage based on a tag" in { // TODO: adding uuids whenever we want a new independent test is not sustainable. If we truly want to test this every - // time we should generate a new "Files" with a new UUIDF. + // time we should generate a new "Files" with a new UUIDF (and other dependencies we want to control). // Alternatively we could normalise the expected values to not care about any generated Ids val newFileUuid = UUID.randomUUID() withUUIDF(newFileUuid) { @@ -492,25 +494,6 @@ class FilesSpec(docker: RemoteStorageDocker) } } - "succeed from remote storage based on latest" in { - val newFileUuid = UUID.randomUUID() - withUUIDF(newFileUuid) { - val source = CopyFileSource(projectRef, NonEmptyList.of(FileId("file1", tag, projectRef))) - val destination = CopyFileDestination(projectRefOrg2, Some(remoteId), None) - - val expectedDestId = project2.base.iri / newFileUuid.toString - val expectedFilename = "myfile.txt" - val expectedAttr = attributes(filename = expectedFilename, projRef = projectRefOrg2, id = newFileUuid) - val expected = mkResource(expectedDestId, projectRefOrg2, diskRev, expectedAttr) - - val actual = files.copyFiles(source, destination).unsafeRunSync() - actual shouldEqual NonEmptyList.of(expected) - - val fetched = files.fetch(FileId(newFileUuid.toString, projectRefOrg2)).accepted - fetched shouldEqual expected - } - } - "reject if the source file doesn't exist" in { val destination = CopyFileDestination(projectRefOrg2, None, None) val source = CopyFileSource(projectRef, NonEmptyList.of(fileIdIri(nxv + "other"))) diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala index 2eafea3b76..5484cbd2bd 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala @@ -9,7 +9,7 @@ import akka.http.scaladsl.model.{StatusCodes, Uri} import akka.http.scaladsl.server.Route import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.kernel.http.MediaTypeDetectorConfig -import ch.epfl.bluebrain.nexus.delta.kernel.utils.ClasspathResourceLoader +import ch.epfl.bluebrain.nexus.delta.kernel.utils.{ClasspathResourceLoader, TransactionalFileCopier} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest.ComputedDigest import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes, FileId, FileRejection} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{contexts => fileContexts, permissions, FileFixtures, Files, FilesConfig} @@ -138,7 +138,8 @@ class FilesRoutesSpec config, FilesConfig(eventLogConfig, MediaTypeDetectorConfig.Empty), remoteDiskStorageClient, - clock + clock, + TransactionalFileCopier.mk() )(uuidF, typedSystem) private val groupDirectives = DeltaSchemeDirectives( diff --git a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/Main.scala b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/Main.scala index d714b961f1..74e66b9e2b 100644 --- a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/Main.scala +++ b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/Main.scala @@ -6,7 +6,7 @@ import akka.http.scaladsl.Http import akka.http.scaladsl.server.Route import akka.util.Timeout import cats.effect.{ExitCode, IO, IOApp} -import ch.epfl.bluebrain.nexus.delta.kernel.utils.CopyFiles +import ch.epfl.bluebrain.nexus.delta.kernel.utils.TransactionalFileCopier import ch.epfl.bluebrain.nexus.storage.Storages.DiskStorage import ch.epfl.bluebrain.nexus.storage.attributes.{AttributesCache, ContentTypeDetector} import ch.epfl.bluebrain.nexus.storage.auth.AuthorizationMethod @@ -61,9 +61,9 @@ object Main extends IOApp { implicit val clock = Clock.systemUTC implicit val contentTypeDetector = new ContentTypeDetector(appConfig.mediaTypeDetector) - val attributesCache: AttributesCache = AttributesCache[AkkaSource] - val validateFile: ValidateFile = ValidateFile.mk(appConfig.storage) - val copyFiles: CopyFiles = CopyFiles.mk() + val attributesCache: AttributesCache = AttributesCache[AkkaSource] + val validateFile: ValidateFile = ValidateFile.mk(appConfig.storage) + val copyFiles: TransactionalFileCopier = TransactionalFileCopier.mk() val storages: Storages[AkkaSource] = new DiskStorage( diff --git a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/Storages.scala b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/Storages.scala index 2ef96c7147..67c880f792 100644 --- a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/Storages.scala +++ b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/Storages.scala @@ -6,7 +6,7 @@ import akka.stream.alpakka.file.scaladsl.Directory import akka.stream.scaladsl.{FileIO, Keep} import cats.data.{EitherT, NonEmptyList} import cats.effect.IO -import ch.epfl.bluebrain.nexus.delta.kernel.utils.{CopyBetween, CopyFiles} +import ch.epfl.bluebrain.nexus.delta.kernel.utils.{CopyBetween, TransactionalFileCopier} import ch.epfl.bluebrain.nexus.storage.File._ import ch.epfl.bluebrain.nexus.storage.Rejection.PathNotFound import ch.epfl.bluebrain.nexus.storage.StorageError.{InternalError, PermissionsFixingFailed} @@ -168,7 +168,7 @@ object Storages { digestConfig: DigestConfig, cache: AttributesCache, validateFile: ValidateFile, - copyFiles: CopyFiles + copyFiles: TransactionalFileCopier )(implicit ec: ExecutionContext, mt: Materializer diff --git a/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/DiskStorageSpec.scala b/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/DiskStorageSpec.scala index 04c21cd2c5..f9c1d33afb 100644 --- a/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/DiskStorageSpec.scala +++ b/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/DiskStorageSpec.scala @@ -10,7 +10,7 @@ import akka.util.ByteString import cats.data.NonEmptyList import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.kernel.http.MediaTypeDetectorConfig -import ch.epfl.bluebrain.nexus.delta.kernel.utils.CopyFiles +import ch.epfl.bluebrain.nexus.delta.kernel.utils.TransactionalFileCopier import ch.epfl.bluebrain.nexus.storage.File.{Digest, FileAttributes} import ch.epfl.bluebrain.nexus.storage.Rejection.{PathAlreadyExists, PathNotFound} import ch.epfl.bluebrain.nexus.storage.StorageError.{PathInvalid, PermissionsFixingFailed} @@ -52,7 +52,7 @@ class DiskStorageSpec val contentTypeDetector = new ContentTypeDetector(MediaTypeDetectorConfig.Empty) val cache = mock[AttributesCache] val validateFile = ValidateFile.mk(sConfig) - val copyFiles = CopyFiles.mk() + val copyFiles = TransactionalFileCopier.mk() val storage = new DiskStorage(sConfig, contentTypeDetector, dConfig, cache, validateFile, copyFiles) override def afterAll(): Unit = { diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/CopyFileSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/CopyFileSpec.scala index f0a310c90b..97896463e1 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/CopyFileSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/CopyFileSpec.scala @@ -7,64 +7,81 @@ import cats.implicits.toTraverseOps import ch.epfl.bluebrain.nexus.delta.kernel.utils.UrlUtils import ch.epfl.bluebrain.nexus.tests.HttpClient._ import ch.epfl.bluebrain.nexus.tests.Identity.storages.Coyote +import ch.epfl.bluebrain.nexus.tests.Optics +import ch.epfl.bluebrain.nexus.tests.kg.CopyFileSpec.Response import io.circe.syntax.KeyOps import io.circe.{Decoder, DecodingFailure, Json, JsonObject} import org.scalatest.Assertion -import scala.concurrent.duration.DurationInt - trait CopyFileSpec { self: StorageSpec => - "Copying multiple files to a different organization" should { + "Copying multiple files" should { - def givenAProjectWithStorage(test: String => IO[Assertion]): IO[Assertion] = { - val (proj, org) = (genId(), genId()) - val projRef = s"$org/$proj" - createProjects(Coyote, org, proj) >> - createStorages(projRef) >> - test(projRef) + "succeed for a project in the same organization" in { + givenANewProjectWithStorage(orgId) { destProjRef => + copyFilesAndCheckSavedResourcesAndContents(destProjRef) + } } - final case class Response(ids: List[String]) - implicit val dec: Decoder[Response] = Decoder.instance { cur => - cur - .get[List[JsonObject]]("_results") - .flatMap(_.traverse(_.apply("@id").flatMap(_.asString).toRight(DecodingFailure("Missing id", Nil)))) - .map(Response) + "succeed for a project in a different organization" in { + givenANewOrgProjectStorage { destProjRef => + copyFilesAndCheckSavedResourcesAndContents(destProjRef) + } } + } - "succeed" in { - givenAProjectWithStorage { destProjRef => - val sourceFiles = List(emptyTextFile, updatedJsonFileWithContentType, textFileWithContentType) - val sourcePayloads = sourceFiles.map(f => Json.obj("sourceFileId" := f.fileId)) + private def copyFilesAndCheckSavedResourcesAndContents(destProjRef: String): IO[Assertion] = { + val sourceFiles = List(emptyTextFile, updatedJsonFileWithContentType, textFileWithContentType) + val sourcePayloads = sourceFiles.map(f => Json.obj("sourceFileId" := f.fileId)) + val payload = Json.obj("sourceProjectRef" := self.projectRef, "files" := sourcePayloads) + val uri = s"/files/$destProjRef?storage=nxv:$storageId" - val payload = Json.obj( - "sourceProjectRef" := self.projectRef, - "files" := sourcePayloads - ) - val uri = s"/files/$destProjRef?storage=nxv:$storageId" + for { + response <- deltaClient.postAndReturn[Response](uri, payload, Coyote) { (json, response) => + (json, expectCreated(json, response)) + } + _ <- checkFileResourcesExist(destProjRef, response) + assertions <- checkFileContentsAreCopiedCorrectly(destProjRef, sourceFiles, response) + } yield assertions.head + } + + private def checkFileContentsAreCopiedCorrectly(destProjRef: String, sourceFiles: List[Input], response: Response) = + response.ids.zip(sourceFiles).traverse { case (destId, Input(_, filename, contentType, contents)) => + deltaClient + .get[ByteString](s"/files/$destProjRef/${UrlUtils.encode(destId)}", Coyote, acceptAll) { + expectDownload(filename, contentType, contents) + } + } - for { - response <- deltaClient.postAndReturn[Response](uri, payload, Coyote) { (json, response) => - (json, expectCreated(json, response)) - } - _ <- IO.sleep(5.seconds) - _ <- response.ids.traverse { id => - deltaClient.get[Json](s"/files/$destProjRef/${UrlUtils.encode(id)}", Coyote) { (json, response) => - println(s"Received json for id $id: $json") - response.status shouldEqual StatusCodes.OK - } - } - assertions <- - response.ids.zip(sourceFiles).traverse { case (destId, Input(_, filename, contentType, contents)) => - println(s"Fetching file $destId") - deltaClient - .get[ByteString](s"/files/$destProjRef/${UrlUtils.encode(destId)}", Coyote, acceptAll) { - expectDownload(filename, contentType, contents) - } - } - } yield assertions.head + private def checkFileResourcesExist(destProjRef: String, response: Response) = + response.ids.traverse { id => + deltaClient.get[Json](s"/files/$destProjRef/${UrlUtils.encode(id)}", Coyote) { (json, response) => + response.status shouldEqual StatusCodes.OK + Optics.`@id`.getOption(json) shouldEqual Some(id) } } + + def givenANewProjectWithStorage(org: String)(test: String => IO[Assertion]): IO[Assertion] = { + val proj = genId() + val projRef = s"$org/$proj" + createProjects(Coyote, org, proj) >> + createStorages(projRef, storageId, storageName) >> + test(projRef) + } + + def givenANewOrgProjectStorage(test: String => IO[Assertion]): IO[Assertion] = + givenANewProjectWithStorage(genId())(test) +} + +object CopyFileSpec { + final case class Response(ids: List[String]) + + object Response { + implicit val dec: Decoder[Response] = Decoder.instance { cur => + cur + .get[List[JsonObject]]("_results") + .flatMap(_.traverse(_.apply("@id").flatMap(_.asString).toRight(DecodingFailure("Missing id", Nil)))) + .map(Response(_)) + } } } diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/DiskStorageSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/DiskStorageSpec.scala index 1ae4a01753..628301aa0a 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/DiskStorageSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/DiskStorageSpec.scala @@ -32,7 +32,7 @@ class DiskStorageSpec extends StorageSpec with CopyFileSpec { ): _* ) - override def createStorages(projectRef: String): IO[Assertion] = { + override def createStorages(projectRef: String, storId: String, storName: String): IO[Assertion] = { val payload = jsonContentOf("kg/storages/disk.json") val payload2 = jsonContentOf("kg/storages/disk-perms.json") @@ -40,21 +40,21 @@ class DiskStorageSpec extends StorageSpec with CopyFileSpec { _ <- deltaClient.post[Json](s"/storages/$projectRef", payload, Coyote) { (_, response) => response.status shouldEqual StatusCodes.Created } - _ <- deltaClient.get[Json](s"/storages/$projectRef/nxv:$storageId", Coyote) { (json, response) => - val expected = storageResponse(projectRef, storageId, "resources/read", "files/write") + _ <- deltaClient.get[Json](s"/storages/$projectRef/nxv:$storId", Coyote) { (json, response) => + val expected = storageResponse(projectRef, storId, "resources/read", "files/write") filterMetadataKeys(json) should equalIgnoreArrayOrder(expected) response.status shouldEqual StatusCodes.OK } _ <- permissionDsl.addPermissions( - Permission(storageName, "read"), - Permission(storageName, "write") + Permission(storName, "read"), + Permission(storName, "write") ) _ <- deltaClient.post[Json](s"/storages/$projectRef", payload2, Coyote) { (_, response) => response.status shouldEqual StatusCodes.Created } - storageId2 = s"${storageId}2" + storageId2 = s"${storId}2" _ <- deltaClient.get[Json](s"/storages/$projectRef/nxv:$storageId2", Coyote) { (json, response) => - val expected = storageResponse(projectRef, storageId2, s"$storageName/read", s"$storageName/write") + val expected = storageResponse(projectRef, storageId2, s"$storName/read", s"$storName/write") filterMetadataKeys(json) should equalIgnoreArrayOrder(expected) response.status shouldEqual StatusCodes.OK } diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/RemoteStorageSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/RemoteStorageSpec.scala index ae9a9f0853..da6184ec0c 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/RemoteStorageSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/RemoteStorageSpec.scala @@ -60,23 +60,23 @@ class RemoteStorageSpec extends StorageSpec with CopyFileSpec { ): _* ) - override def createStorages(projectRef: String): IO[Assertion] = { + override def createStorages(projectRef: String, storId: String, storName: String): IO[Assertion] = { val payload = jsonContentOf( "kg/storages/remote-disk.json", "endpoint" -> externalEndpoint, "read" -> "resources/read", "write" -> "files/write", "folder" -> remoteFolder, - "id" -> storageId + "id" -> storId ) val payload2 = jsonContentOf( "kg/storages/remote-disk.json", "endpoint" -> externalEndpoint, - "read" -> s"$storageName/read", - "write" -> s"$storageName/write", + "read" -> s"$storName/read", + "write" -> s"$storName/write", "folder" -> remoteFolder, - "id" -> s"${storageId}2" + "id" -> s"${storId}2" ) for { @@ -85,12 +85,12 @@ class RemoteStorageSpec extends StorageSpec with CopyFileSpec { fail(s"Unexpected status '${response.status}', response:\n${json.spaces2}") } else succeed } - _ <- deltaClient.get[Json](s"/storages/$projectRef/nxv:$storageId", Coyote) { (json, response) => - val expected = storageResponse(projectRef, storageId, "resources/read", "files/write") + _ <- deltaClient.get[Json](s"/storages/$projectRef/nxv:$storId", Coyote) { (json, response) => + val expected = storageResponse(projectRef, storId, "resources/read", "files/write") filterMetadataKeys(json) should equalIgnoreArrayOrder(expected) response.status shouldEqual StatusCodes.OK } - _ <- deltaClient.get[Json](s"/storages/$projectRef/nxv:$storageId/source", Coyote) { (json, response) => + _ <- deltaClient.get[Json](s"/storages/$projectRef/nxv:$storId/source", Coyote) { (json, response) => response.status shouldEqual StatusCodes.OK val expected = jsonContentOf( "kg/storages/storage-source.json", @@ -107,7 +107,7 @@ class RemoteStorageSpec extends StorageSpec with CopyFileSpec { _ <- deltaClient.post[Json](s"/storages/$projectRef", payload2, Coyote) { (_, response) => response.status shouldEqual StatusCodes.Created } - storageId2 = s"${storageId}2" + storageId2 = s"${storId}2" _ <- deltaClient.get[Json](s"/storages/$projectRef/nxv:$storageId2", Coyote) { (json, response) => val expected = storageResponse(projectRef, storageId2, s"$storageName/read", s"$storageName/write") filterMetadataKeys(json) should equalIgnoreArrayOrder(expected) diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/S3StorageSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/S3StorageSpec.scala index d461207fa7..00739d115c 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/S3StorageSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/S3StorageSpec.scala @@ -82,39 +82,39 @@ class S3StorageSpec extends StorageSpec { ): _* ) - override def createStorages(projectRef: String): IO[Assertion] = { + override def createStorages(projectRef: String, storId: String, storName: String): IO[Assertion] = { val payload = jsonContentOf( "kg/storages/s3.json", - "storageId" -> s"https://bluebrain.github.io/nexus/vocabulary/$storageId", + "storageId" -> s"https://bluebrain.github.io/nexus/vocabulary/$storId", "bucket" -> bucket, "endpoint" -> s3Endpoint ) val payload2 = jsonContentOf( "kg/storages/s3.json", - "storageId" -> s"https://bluebrain.github.io/nexus/vocabulary/${storageId}2", + "storageId" -> s"https://bluebrain.github.io/nexus/vocabulary/${storId}2", "bucket" -> bucket, "endpoint" -> s3Endpoint ) deepMerge Json.obj( "region" -> Json.fromString("eu-west-2"), - "readPermission" -> Json.fromString(s"$storageName/read"), - "writePermission" -> Json.fromString(s"$storageName/write") + "readPermission" -> Json.fromString(s"$storName/read"), + "writePermission" -> Json.fromString(s"$storName/write") ) for { _ <- deltaClient.post[Json](s"/storages/$projectRef", payload, Coyote) { (_, response) => response.status shouldEqual StatusCodes.Created } - _ <- deltaClient.get[Json](s"/storages/$projectRef/nxv:$storageId", Coyote) { (json, response) => - val expected = storageResponse(projectRef, storageId, "resources/read", "files/write") + _ <- deltaClient.get[Json](s"/storages/$projectRef/nxv:$storId", Coyote) { (json, response) => + val expected = storageResponse(projectRef, storId, "resources/read", "files/write") filterMetadataKeys(json) should equalIgnoreArrayOrder(expected) response.status shouldEqual StatusCodes.OK } - _ <- permissionDsl.addPermissions(Permission(storageName, "read"), Permission(storageName, "write")) + _ <- permissionDsl.addPermissions(Permission(storName, "read"), Permission(storName, "write")) _ <- deltaClient.post[Json](s"/storages/$projectRef", payload2, Coyote) { (_, response) => response.status shouldEqual StatusCodes.Created } - storageId2 = s"${storageId}2" + storageId2 = s"${storId}2" _ <- deltaClient.get[Json](s"/storages/$projectRef/nxv:$storageId2", Coyote) { (json, response) => val expected = storageResponse(projectRef, storageId2, "s3/read", "s3/write") .deepMerge(Json.obj("region" -> Json.fromString("eu-west-2"))) diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/StorageSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/StorageSpec.scala index a1316e500e..78c8bc5db5 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/StorageSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/StorageSpec.scala @@ -40,7 +40,7 @@ abstract class StorageSpec extends BaseIntegrationSpec { def locationPrefix: Option[String] - def createStorages(projectRef: String): IO[Assertion] + def createStorages(projectRef: String, storId: String, storName: String): IO[Assertion] protected def fileSelf(project: String, id: String): String = { val uri = Uri(s"${config.deltaUri}/files/$project") @@ -68,7 +68,7 @@ abstract class StorageSpec extends BaseIntegrationSpec { "Creating a storage" should { s"succeed for a $storageName storage" in { - createStorages(projectRef) + createStorages(projectRef, storageId, storageName) } "wait for storages to be indexed" in { @@ -399,10 +399,18 @@ abstract class StorageSpec extends BaseIntegrationSpec { } } - private def uploadFile(fileInput: Input, rev: Option[Int]): ((Json, HttpResponse) => Assertion) => IO[Assertion] = { + def uploadFile(fileInput: Input, rev: Option[Int]): ((Json, HttpResponse) => Assertion) => IO[Assertion] = + uploadFileToProjectStorage(fileInput, projectRef, storageId, rev) + + def uploadFileToProjectStorage( + fileInput: Input, + projRef: String, + storage: String, + rev: Option[Int] + ): ((Json, HttpResponse) => Assertion) => IO[Assertion] = { val revString = rev.map(r => s"&rev=$r").getOrElse("") deltaClient.uploadFile[Json]( - s"/files/$projectRef/${fileInput.fileId}?storage=nxv:$storageId$revString", + s"/files/$projRef/${fileInput.fileId}?storage=nxv:$storage$revString", fileInput.contents, fileInput.ct, fileInput.filename, From a366c8393b4504c2c1ce15d327ccfff1c98d5d66 Mon Sep 17 00:00:00 2001 From: dantb Date: Thu, 7 Dec 2023 17:43:28 +0100 Subject: [PATCH 07/18] Add test with source files in different storages --- .../utils/TransactionalFileCopier.scala | 5 +- .../disk/DiskStorageFetchFile.scala | 6 +- .../resources/contexts/bulk-operation.json | 1 - .../sdk/jsonld/BulkOperationResults.scala | 5 +- .../resources/kg/storages/disk-perms.json | 6 +- .../src/test/resources/kg/storages/disk.json | 2 +- .../resources/kg/storages/storage-source.json | 2 +- .../nexus/tests/kg/CopyFileSpec.scala | 75 +++++++++++++---- .../nexus/tests/kg/DiskStorageSpec.scala | 49 ++++++------ .../nexus/tests/kg/RemoteStorageSpec.scala | 80 ++++++++++--------- 10 files changed, 138 insertions(+), 93 deletions(-) diff --git a/delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/TransactionalFileCopier.scala b/delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/TransactionalFileCopier.scala index 49dba83df7..254d5b5273 100644 --- a/delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/TransactionalFileCopier.scala +++ b/delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/TransactionalFileCopier.scala @@ -32,8 +32,9 @@ object TransactionalFileCopier { } .void .handleErrorWith { e => - logger.error(e)("Transactional files copy failed, deleting created files") >> - rollbackCopiesAndRethrow(errorRef, files.map(_.destination)) + val destinations = files.map(_.destination) + logger.error(e)(s"Transactional files copy failed, deleting created files: ${destinations}") >> + rollbackCopiesAndRethrow(errorRef, destinations) } } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageFetchFile.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageFetchFile.scala index 8f80e4b238..537a7a9925 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageFetchFile.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageFetchFile.scala @@ -18,7 +18,9 @@ object DiskStorageFetchFile extends FetchFile { absoluteDiskPath(path).redeemWith( e => IO.raiseError(UnexpectedLocationFormat(s"file://$path", e.getMessage)), path => - IO.raiseWhen(!path.toFile.exists())(FetchFileRejection.FileNotFound(path.toString)) - .map(_ => FileIO.fromPath(path)) + IO(path.toFile.exists()).flatMap { exists => + if (exists) IO(FileIO.fromPath(path)) + else IO.raiseError(FetchFileRejection.FileNotFound(path.toString)) + } ) } diff --git a/delta/sdk/src/main/resources/contexts/bulk-operation.json b/delta/sdk/src/main/resources/contexts/bulk-operation.json index 426ed55846..6be92c1da4 100644 --- a/delta/sdk/src/main/resources/contexts/bulk-operation.json +++ b/delta/sdk/src/main/resources/contexts/bulk-operation.json @@ -1,6 +1,5 @@ { "@context": { - "_total": "https://bluebrain.github.io/nexus/vocabulary/total", "_results": { "@id": "https://bluebrain.github.io/nexus/vocabulary/results", "@container": "@list" diff --git a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/jsonld/BulkOperationResults.scala b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/jsonld/BulkOperationResults.scala index e060200843..da62735072 100644 --- a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/jsonld/BulkOperationResults.scala +++ b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/jsonld/BulkOperationResults.scala @@ -14,10 +14,7 @@ object BulkOperationResults { implicit def encoder[A: Encoder.AsObject]: Encoder.AsObject[BulkOperationResults[A]] = Encoder.AsObject.instance { r => - JsonObject( - nxv.total.prefix -> Json.fromInt(r.results.size), - nxv.results.prefix -> Json.fromValues(r.results.map(_.asJson)) - ) + JsonObject(nxv.results.prefix -> Json.fromValues(r.results.map(_.asJson))) } def searchResultsJsonLdEncoder[A: Encoder.AsObject]( diff --git a/tests/src/test/resources/kg/storages/disk-perms.json b/tests/src/test/resources/kg/storages/disk-perms.json index 7f7a6ebc15..9f17d640c6 100644 --- a/tests/src/test/resources/kg/storages/disk-perms.json +++ b/tests/src/test/resources/kg/storages/disk-perms.json @@ -1,8 +1,8 @@ { - "@id": "https://bluebrain.github.io/nexus/vocabulary/mystorage2", + "@id": "https://bluebrain.github.io/nexus/vocabulary/{{id}}", "@type": "DiskStorage", "volume": "/default-volume", "default": false, - "readPermission": "disk/read", - "writePermission": "disk/write" + "readPermission": "{{read}}", + "writePermission": "{{write}}" } \ No newline at end of file diff --git a/tests/src/test/resources/kg/storages/disk.json b/tests/src/test/resources/kg/storages/disk.json index e9b286f0e4..a2451b08a1 100644 --- a/tests/src/test/resources/kg/storages/disk.json +++ b/tests/src/test/resources/kg/storages/disk.json @@ -1,5 +1,5 @@ { - "@id": "https://bluebrain.github.io/nexus/vocabulary/mystorage", + "@id": "https://bluebrain.github.io/nexus/vocabulary/{{id}}", "@type": "DiskStorage", "volume": "/default-volume", "default": false diff --git a/tests/src/test/resources/kg/storages/storage-source.json b/tests/src/test/resources/kg/storages/storage-source.json index e589fe675b..0b32544327 100644 --- a/tests/src/test/resources/kg/storages/storage-source.json +++ b/tests/src/test/resources/kg/storages/storage-source.json @@ -1,5 +1,5 @@ { - "@id" : "https://bluebrain.github.io/nexus/vocabulary/myexternalstorage", + "@id" : "https://bluebrain.github.io/nexus/vocabulary/{{id}}", "@type" : "RemoteDiskStorage", "default" : false, "endpoint" : "{{storageBase}}", diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/CopyFileSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/CopyFileSpec.scala index 97896463e1..edca249505 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/CopyFileSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/CopyFileSpec.scala @@ -1,6 +1,6 @@ package ch.epfl.bluebrain.nexus.tests.kg -import akka.http.scaladsl.model.StatusCodes +import akka.http.scaladsl.model.{ContentTypes, StatusCodes} import akka.util.ByteString import cats.effect.IO import cats.implicits.toTraverseOps @@ -8,7 +8,7 @@ import ch.epfl.bluebrain.nexus.delta.kernel.utils.UrlUtils import ch.epfl.bluebrain.nexus.tests.HttpClient._ import ch.epfl.bluebrain.nexus.tests.Identity.storages.Coyote import ch.epfl.bluebrain.nexus.tests.Optics -import ch.epfl.bluebrain.nexus.tests.kg.CopyFileSpec.Response +import ch.epfl.bluebrain.nexus.tests.kg.CopyFileSpec.{Response, StorageDetails} import io.circe.syntax.KeyOps import io.circe.{Decoder, DecodingFailure, Json, JsonObject} import org.scalatest.Assertion @@ -18,23 +18,57 @@ trait CopyFileSpec { self: StorageSpec => "Copying multiple files" should { "succeed for a project in the same organization" in { - givenANewProjectWithStorage(orgId) { destProjRef => - copyFilesAndCheckSavedResourcesAndContents(destProjRef) + givenANewProjectAndStorageInExistingOrg(orgId) { destStorage => + val existingFiles = List(emptyTextFile, updatedJsonFileWithContentType, textFileWithContentType) + copyFilesAndCheckSavedResourcesAndContents(projectRef, existingFiles, destStorage) } } "succeed for a project in a different organization" in { - givenANewOrgProjectStorage { destProjRef => - copyFilesAndCheckSavedResourcesAndContents(destProjRef) + givenANewOrgProjectAndStorage { destStorage => + val sourceFiles = List(emptyTextFile, updatedJsonFileWithContentType, textFileWithContentType) + copyFilesAndCheckSavedResourcesAndContents(projectRef, sourceFiles, destStorage) } } + + "succeed for source files in different storages within a project" in { + givenANewStorageInExistingProject(projectRef) { sourceStorage1 => + givenANewStorageInExistingProject(projectRef) { sourceStorage2 => + givenANewOrgProjectAndStorage { destStorage => + val (sourceFile1, sourceFile2) = (genTextFileInput(), genTextFileInput()) + val sourceFiles = List(sourceFile1, sourceFile2) + + for { + _ <- uploadFileToProjectStorage(sourceFile1, sourceStorage1.projRef, sourceStorage1.storageId, None)( + expectCreated + ) + _ <- uploadFileToProjectStorage(sourceFile2, sourceStorage2.projRef, sourceStorage2.storageId, None)( + expectCreated + ) + result <- copyFilesAndCheckSavedResourcesAndContents(projectRef, sourceFiles, destStorage) + } yield result + } + } + } + } + } - private def copyFilesAndCheckSavedResourcesAndContents(destProjRef: String): IO[Assertion] = { - val sourceFiles = List(emptyTextFile, updatedJsonFileWithContentType, textFileWithContentType) + def genTextFileInput(): Input = Input(genId(), genString(), ContentTypes.`text/plain(UTF-8)`, genString()) + + def mkPayload(sourceProjRef: String, sourceFiles: List[Input]): Json = { val sourcePayloads = sourceFiles.map(f => Json.obj("sourceFileId" := f.fileId)) - val payload = Json.obj("sourceProjectRef" := self.projectRef, "files" := sourcePayloads) - val uri = s"/files/$destProjRef?storage=nxv:$storageId" + Json.obj("sourceProjectRef" := sourceProjRef, "files" := sourcePayloads) + } + + private def copyFilesAndCheckSavedResourcesAndContents( + sourceProjRef: String, + sourceFiles: List[Input], + destStorage: StorageDetails + ): IO[Assertion] = { + val destProjRef = destStorage.projRef + val payload = mkPayload(sourceProjRef, sourceFiles) + val uri = s"/files/$destProjRef?storage=nxv:${destStorage.storageId}" for { response <- deltaClient.postAndReturn[Response](uri, payload, Coyote) { (json, response) => @@ -45,7 +79,7 @@ trait CopyFileSpec { self: StorageSpec => } yield assertions.head } - private def checkFileContentsAreCopiedCorrectly(destProjRef: String, sourceFiles: List[Input], response: Response) = + def checkFileContentsAreCopiedCorrectly(destProjRef: String, sourceFiles: List[Input], response: Response) = response.ids.zip(sourceFiles).traverse { case (destId, Input(_, filename, contentType, contents)) => deltaClient .get[ByteString](s"/files/$destProjRef/${UrlUtils.encode(destId)}", Coyote, acceptAll) { @@ -53,7 +87,7 @@ trait CopyFileSpec { self: StorageSpec => } } - private def checkFileResourcesExist(destProjRef: String, response: Response) = + def checkFileResourcesExist(destProjRef: String, response: Response) = response.ids.traverse { id => deltaClient.get[Json](s"/files/$destProjRef/${UrlUtils.encode(id)}", Coyote) { (json, response) => response.status shouldEqual StatusCodes.OK @@ -61,19 +95,26 @@ trait CopyFileSpec { self: StorageSpec => } } - def givenANewProjectWithStorage(org: String)(test: String => IO[Assertion]): IO[Assertion] = { + def givenANewProjectAndStorageInExistingOrg(org: String)(test: StorageDetails => IO[Assertion]): IO[Assertion] = { val proj = genId() val projRef = s"$org/$proj" createProjects(Coyote, org, proj) >> - createStorages(projRef, storageId, storageName) >> - test(projRef) + givenANewStorageInExistingProject(projRef)(test) } - def givenANewOrgProjectStorage(test: String => IO[Assertion]): IO[Assertion] = - givenANewProjectWithStorage(genId())(test) + def givenANewStorageInExistingProject(projRef: String)(test: StorageDetails => IO[Assertion]): IO[Assertion] = { + val (storageId, storageName) = (genId(), genString()) + createStorages(projRef, storageId, storageName) >> + test(StorageDetails(projRef, storageId)) + } + + def givenANewOrgProjectAndStorage(test: StorageDetails => IO[Assertion]): IO[Assertion] = + givenANewProjectAndStorageInExistingOrg(genId())(test) } object CopyFileSpec { + final case class StorageDetails(projRef: String, storageId: String) + final case class Response(ids: List[String]) object Response { diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/DiskStorageSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/DiskStorageSpec.scala index 628301aa0a..887982c97e 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/DiskStorageSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/DiskStorageSpec.scala @@ -33,31 +33,34 @@ class DiskStorageSpec extends StorageSpec with CopyFileSpec { ) override def createStorages(projectRef: String, storId: String, storName: String): IO[Assertion] = { - val payload = jsonContentOf("kg/storages/disk.json") - val payload2 = jsonContentOf("kg/storages/disk-perms.json") + val payload = jsonContentOf("kg/storages/disk.json", "id" -> storId) + val storageId2 = s"${storId}2" + val storage2Read = s"$storName/read" + val storage2Write = s"$storName/write" + val payload2 = + jsonContentOf("kg/storages/disk-perms.json", "id" -> storageId2, "read" -> storage2Read, "write" -> storage2Write) for { - _ <- deltaClient.post[Json](s"/storages/$projectRef", payload, Coyote) { (_, response) => - response.status shouldEqual StatusCodes.Created - } - _ <- deltaClient.get[Json](s"/storages/$projectRef/nxv:$storId", Coyote) { (json, response) => - val expected = storageResponse(projectRef, storId, "resources/read", "files/write") - filterMetadataKeys(json) should equalIgnoreArrayOrder(expected) - response.status shouldEqual StatusCodes.OK - } - _ <- permissionDsl.addPermissions( - Permission(storName, "read"), - Permission(storName, "write") - ) - _ <- deltaClient.post[Json](s"/storages/$projectRef", payload2, Coyote) { (_, response) => - response.status shouldEqual StatusCodes.Created - } - storageId2 = s"${storId}2" - _ <- deltaClient.get[Json](s"/storages/$projectRef/nxv:$storageId2", Coyote) { (json, response) => - val expected = storageResponse(projectRef, storageId2, s"$storName/read", s"$storName/write") - filterMetadataKeys(json) should equalIgnoreArrayOrder(expected) - response.status shouldEqual StatusCodes.OK - } + _ <- deltaClient.post[Json](s"/storages/$projectRef", payload, Coyote) { (_, response) => + response.status shouldEqual StatusCodes.Created + } + _ <- deltaClient.get[Json](s"/storages/$projectRef/nxv:$storId", Coyote) { (json, response) => + val expected = storageResponse(projectRef, storId, "resources/read", "files/write") + filterMetadataKeys(json) should equalIgnoreArrayOrder(expected) + response.status shouldEqual StatusCodes.OK + } + _ <- permissionDsl.addPermissions( + Permission(storName, "read"), + Permission(storName, "write") + ) + _ <- deltaClient.post[Json](s"/storages/$projectRef", payload2, Coyote) { (_, response) => + response.status shouldEqual StatusCodes.Created + } + _ <- deltaClient.get[Json](s"/storages/$projectRef/nxv:$storageId2", Coyote) { (json, response) => + val expected = storageResponse(projectRef, storageId2, storage2Read, storage2Write) + filterMetadataKeys(json) should equalIgnoreArrayOrder(expected) + response.status shouldEqual StatusCodes.OK + } } yield succeed } diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/RemoteStorageSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/RemoteStorageSpec.scala index da6184ec0c..c6bec7aae0 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/RemoteStorageSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/RemoteStorageSpec.scala @@ -61,7 +61,7 @@ class RemoteStorageSpec extends StorageSpec with CopyFileSpec { ) override def createStorages(projectRef: String, storId: String, storName: String): IO[Assertion] = { - val payload = jsonContentOf( + val payload = jsonContentOf( "kg/storages/remote-disk.json", "endpoint" -> externalEndpoint, "read" -> "resources/read", @@ -69,50 +69,52 @@ class RemoteStorageSpec extends StorageSpec with CopyFileSpec { "folder" -> remoteFolder, "id" -> storId ) - - val payload2 = jsonContentOf( + val storageId2 = s"${storId}2" + val storage2Read = s"$storName/read" + val storage2Write = s"$storName/write" + val payload2 = jsonContentOf( "kg/storages/remote-disk.json", "endpoint" -> externalEndpoint, - "read" -> s"$storName/read", - "write" -> s"$storName/write", + "read" -> storage2Read, + "write" -> storage2Write, "folder" -> remoteFolder, - "id" -> s"${storId}2" + "id" -> storageId2 ) for { - _ <- deltaClient.post[Json](s"/storages/$projectRef", payload, Coyote) { (json, response) => - if (response.status != StatusCodes.Created) { - fail(s"Unexpected status '${response.status}', response:\n${json.spaces2}") - } else succeed - } - _ <- deltaClient.get[Json](s"/storages/$projectRef/nxv:$storId", Coyote) { (json, response) => - val expected = storageResponse(projectRef, storId, "resources/read", "files/write") - filterMetadataKeys(json) should equalIgnoreArrayOrder(expected) - response.status shouldEqual StatusCodes.OK - } - _ <- deltaClient.get[Json](s"/storages/$projectRef/nxv:$storId/source", Coyote) { (json, response) => - response.status shouldEqual StatusCodes.OK - val expected = jsonContentOf( - "kg/storages/storage-source.json", - "folder" -> remoteFolder, - "storageBase" -> externalEndpoint - ) - filterKey("credentials")(json) should equalIgnoreArrayOrder(expected) - - } - _ <- permissionDsl.addPermissions( - Permission(storageName, "read"), - Permission(storageName, "write") - ) - _ <- deltaClient.post[Json](s"/storages/$projectRef", payload2, Coyote) { (_, response) => - response.status shouldEqual StatusCodes.Created - } - storageId2 = s"${storId}2" - _ <- deltaClient.get[Json](s"/storages/$projectRef/nxv:$storageId2", Coyote) { (json, response) => - val expected = storageResponse(projectRef, storageId2, s"$storageName/read", s"$storageName/write") - filterMetadataKeys(json) should equalIgnoreArrayOrder(expected) - response.status shouldEqual StatusCodes.OK - } + _ <- deltaClient.post[Json](s"/storages/$projectRef", payload, Coyote) { (json, response) => + if (response.status != StatusCodes.Created) { + fail(s"Unexpected status '${response.status}', response:\n${json.spaces2}") + } else succeed + } + _ <- deltaClient.get[Json](s"/storages/$projectRef/nxv:$storId", Coyote) { (json, response) => + val expected = storageResponse(projectRef, storId, "resources/read", "files/write") + filterMetadataKeys(json) should equalIgnoreArrayOrder(expected) + response.status shouldEqual StatusCodes.OK + } + _ <- deltaClient.get[Json](s"/storages/$projectRef/nxv:$storId/source", Coyote) { (json, response) => + response.status shouldEqual StatusCodes.OK + val expected = jsonContentOf( + "kg/storages/storage-source.json", + "folder" -> remoteFolder, + "storageBase" -> externalEndpoint, + "id" -> storId + ) + filterKey("credentials")(json) should equalIgnoreArrayOrder(expected) + + } + _ <- permissionDsl.addPermissions( + Permission(storName, "read"), + Permission(storName, "write") + ) + _ <- deltaClient.post[Json](s"/storages/$projectRef", payload2, Coyote) { (_, response) => + response.status shouldEqual StatusCodes.Created + } + _ <- deltaClient.get[Json](s"/storages/$projectRef/nxv:$storageId2", Coyote) { (json, response) => + val expected = storageResponse(projectRef, storageId2, storage2Read, storage2Write) + filterMetadataKeys(json) should equalIgnoreArrayOrder(expected) + response.status shouldEqual StatusCodes.OK + } } yield succeed } From 2291cdf72b7b449f895f847892ae8f1fa5075b51 Mon Sep 17 00:00:00 2001 From: dantb Date: Thu, 7 Dec 2023 19:44:04 +0100 Subject: [PATCH 08/18] Extract bulk file operations into new package --- .../delta/plugins/storage/files/Files.scala | 47 +++----- .../storage/files/batch/BatchCopy.scala | 107 ++++++++++++++++++ .../storage/files/batch/BatchFiles.scala | 69 +++++++++++ .../storage/files/model/FileRejection.scala | 7 -- .../storages/operations/disk/DiskCopy.scala | 56 +++++++++ .../operations/disk/DiskCopyDetails.scala | 10 ++ .../operations/remote/RemoteDiskCopy.scala | 43 +++++++ .../remote/RemoteDiskCopyDetails.scala | 12 ++ 8 files changed, 315 insertions(+), 36 deletions(-) create mode 100644 delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopy.scala create mode 100644 delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFiles.scala create mode 100644 delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskCopy.scala create mode 100644 delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskCopyDetails.scala create mode 100644 delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskCopy.scala create mode 100644 delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskCopyDetails.scala diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala index 9a9c89202b..e5493cf54b 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala @@ -461,7 +461,20 @@ final class Files( * @param project * the project where the storage belongs */ - def fetch(id: FileId): IO[FileResource] = Files.fetch(fetchContext, log)(id).span("fetchFile") + def fetch(id: FileId): IO[FileResource] = + (for { + (iri, _) <- id.expandIri(fetchContext.onRead) + state <- fetchState(id, iri) + } yield state.toResource).span("fetchFile") + + private def fetchState(id: FileId, iri: Iri): IO[FileState] = { + val notFound = FileNotFound(iri, id.project) + id.id match { + case Latest(_) => log.stateOr(id.project, iri, notFound) + case Revision(_, rev) => log.stateOr(id.project, iri, rev, notFound, RevisionNotFound) + case Tag(_, tag) => log.stateOr(id.project, iri, tag, notFound, TagNotFound(tag)) + } + } private def createLink( iri: Iri, @@ -487,12 +500,12 @@ final class Files( .apply(path, desc) .adaptError { case e: StorageFileRejection => LinkRejection(fileId, storage.id, e) } - private def eval(cmd: FileCommand): IO[FileResource] = + def eval(cmd: FileCommand): IO[FileResource] = log.evaluate(cmd.project, cmd.id, cmd).map(_._2.toResource) private def test(cmd: FileCommand) = log.dryRun(cmd.project, cmd.id, cmd) - private def fetchActiveStorage(storageIdOpt: Option[IdSegment], ref: ProjectRef, pc: ProjectContext)(implicit + def fetchActiveStorage(storageIdOpt: Option[IdSegment], ref: ProjectRef, pc: ProjectContext)(implicit caller: Caller ): IO[(ResourceRef.Revision, Storage)] = storageIdOpt match { @@ -512,7 +525,7 @@ final class Files( } yield ResourceRef.Revision(storage.id, storage.rev) -> storage.value } - private def validateAuth(project: ProjectRef, permission: Permission)(implicit c: Caller): IO[Unit] = + def validateAuth(project: ProjectRef, permission: Permission)(implicit c: Caller): IO[Unit] = aclCheck.authorizeForOr(project, permission)(AuthorizationFailed(project, permission)) private def extractFileAttributes(iri: Iri, entity: HttpEntity, storage: Storage): IO[FileAttributes] = @@ -537,7 +550,7 @@ final class Files( WrappedStorageRejection(s) } - private def generateId(pc: ProjectContext)(implicit uuidF: UUIDF): IO[Iri] = + def generateId(pc: ProjectContext)(implicit uuidF: UUIDF): IO[Iri] = uuidF().map(uuid => pc.base.iri / uuid.toString) /** @@ -859,28 +872,4 @@ object Files { ) .void } - - /** - * Fetch the last version of a file - * - * @param id - * the identifier that will be expanded to the Iri of the file with its optional rev/tag - * @param project - * the project where the storage belongs - */ - def fetch(fetchContext: FetchContext[FileRejection], log: FilesLog)(id: FileId): IO[FileResource] = - for { - (iri, _) <- id.expandIri(fetchContext.onRead) - state <- fetchState(log)(id, iri) - } yield state.toResource - - private def fetchState(log: FilesLog)(id: FileId, iri: Iri): IO[FileState] = { - val notFound = FileNotFound(iri, id.project) - id.id match { - case Latest(_) => log.stateOr(id.project, iri, notFound) - case Revision(_, rev) => log.stateOr(id.project, iri, rev, notFound, RevisionNotFound) - case Tag(_, tag) => log.stateOr(id.project, iri, tag, notFound, TagNotFound(tag)) - } - } - } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopy.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopy.scala new file mode 100644 index 0000000000..8105a50859 --- /dev/null +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopy.scala @@ -0,0 +1,107 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch + +import cats.data.NonEmptyList +import cats.effect.IO +import cats.implicits.toFunctorOps +import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.Files +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection._ +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model._ +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.CopyFileSource +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.{DiskStorage, RemoteDiskStorage} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageRejection.DifferentStorageType +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{Storage, StorageType} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.CopyFileRejection +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.{DiskCopy, DiskCopyDetails} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.{RemoteDiskCopy, RemoteDiskCopyDetails} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{Storages, StoragesStatistics} +import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller +import ch.epfl.bluebrain.nexus.delta.sourcing.model.Label +import shapeless.syntax.typeable.typeableOps + +trait BatchCopy { + def copyFiles(source: CopyFileSource, destStorage: Storage)(implicit + c: Caller + ): IO[NonEmptyList[FileAttributes]] +} + +object BatchCopy { + def mk( + files: Files, + storages: Storages, + storagesStatistics: StoragesStatistics, + diskCopy: DiskCopy, + remoteDiskCopy: RemoteDiskCopy + )(implicit uuidF: UUIDF): BatchCopy = new BatchCopy { + + override def copyFiles(source: CopyFileSource, destStorage: Storage)(implicit + c: Caller + ): IO[NonEmptyList[FileAttributes]] = + destStorage match { + case disk: Storage.DiskStorage => copyToDiskStorage(source, disk) + case remote: Storage.RemoteDiskStorage => copyToRemoteStorage(source, remote) + case s3: Storage.S3Storage => unsupported(s3.tpe) + } + + private def copyToRemoteStorage(source: CopyFileSource, remote: RemoteDiskStorage)(implicit c: Caller) = + for { + remoteCopyDetails <- source.files.traverse(fetchRemoteCopyDetails(remote, _)) + _ <- validateSpaceOnStorage(remote, remoteCopyDetails.map(_.sourceAttributes.bytes)) + attributes <- remoteDiskCopy.copyFiles(remoteCopyDetails) + } yield attributes + + private def copyToDiskStorage(source: CopyFileSource, disk: DiskStorage)(implicit c: Caller) = + for { + diskCopyDetails <- source.files.traverse(fetchDiskCopyDetails(disk, _)) + _ <- validateSpaceOnStorage(disk, diskCopyDetails.map(_.sourceAttributes.bytes)) + attributes <- diskCopy.copyFiles(diskCopyDetails) + } yield attributes + + private def validateSpaceOnStorage(destStorage: Storage, sourcesBytes: NonEmptyList[Long]): IO[Unit] = for { + space <- storagesStatistics.getStorageAvailableSpace(destStorage) + maxSize = destStorage.storageValue.maxFileSize + _ <- IO.raiseWhen(sourcesBytes.exists(_ > maxSize))(FileTooLarge(maxSize, space)) + totalSize = sourcesBytes.toList.sum + _ <- IO.raiseWhen(space.exists(_ < totalSize))(FileTooLarge(maxSize, space)) + } yield () + + private def fetchDiskCopyDetails(destStorage: DiskStorage, fileId: FileId)(implicit c: Caller) = + for { + (file, sourceStorage) <- fetchSourceFileAndStorage(fileId) + destinationDesc <- FileDescription(file.attributes.filename, file.attributes.mediaType) + _ <- validateDiskStorage(destStorage, sourceStorage) + } yield DiskCopyDetails(destStorage, destinationDesc, file.attributes) + + private def validateDiskStorage(destStorage: DiskStorage, sourceStorage: Storage) = + sourceStorage + .narrowTo[DiskStorage] + .as(IO.unit) + .getOrElse(IO.raiseError(differentStorageTypeError(destStorage, sourceStorage))) + + private def fetchRemoteCopyDetails(destStorage: RemoteDiskStorage, fileId: FileId)(implicit c: Caller) = + for { + (file, sourceStorage) <- fetchSourceFileAndStorage(fileId) + destinationDesc <- FileDescription(file.attributes.filename, file.attributes.mediaType) + sourceBucket <- validateRemoteStorage(destStorage, sourceStorage) + } yield RemoteDiskCopyDetails(destStorage, destinationDesc, sourceBucket, file.attributes) + + private def validateRemoteStorage(destStorage: RemoteDiskStorage, sourceStorage: Storage) = + sourceStorage + .narrowTo[RemoteDiskStorage] + .map(remote => IO.pure(remote.value.folder)) + .getOrElse(IO.raiseError[Label](differentStorageTypeError(destStorage, sourceStorage))) + + private def differentStorageTypeError(destStorage: Storage, sourceStorage: Storage) = + DifferentStorageType(destStorage.id, found = sourceStorage.tpe, expected = destStorage.tpe) + + private def unsupported(tpe: StorageType) = IO.raiseError(CopyFileRejection.UnsupportedOperation(tpe)) + + private def fetchSourceFileAndStorage(id: FileId)(implicit c: Caller) = + for { + file <- files.fetch(id) + sourceStorage <- storages.fetch(file.value.storage, id.project) + _ <- files.validateAuth(id.project, sourceStorage.value.storageValue.readPermission) + } yield (file.value, sourceStorage.value) + } + +} diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFiles.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFiles.scala new file mode 100644 index 0000000000..512186f062 --- /dev/null +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFiles.scala @@ -0,0 +1,69 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch + +import cats.data.NonEmptyList +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.kernel.Logger +import ch.epfl.bluebrain.nexus.delta.kernel.kamon.KamonMetricComponent +import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.Files.entityType +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileCommand._ +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model._ +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.CopyFileSource +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{FileResource, Files} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageType +import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller +import ch.epfl.bluebrain.nexus.delta.sdk.implicits._ +import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContext +import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.ProjectContext +import ch.epfl.bluebrain.nexus.delta.sourcing.model.ResourceRef + +trait BatchFiles { + def copyFiles( + source: CopyFileSource, + dest: CopyFileDestination + )(implicit c: Caller): IO[NonEmptyList[FileResource]] +} + +object BatchFiles { + def mk(files: Files, fetchContext: FetchContext[FileRejection], batchCopy: BatchCopy)(implicit + uuidF: UUIDF + ): BatchFiles = new BatchFiles { + + private val logger = Logger[BatchFiles] + + implicit private val kamonComponent: KamonMetricComponent = KamonMetricComponent(entityType.value) + + override def copyFiles(source: CopyFileSource, dest: CopyFileDestination)(implicit + c: Caller + ): IO[NonEmptyList[FileResource]] = { + for { + pc <- fetchContext.onCreate(dest.project) + (destStorageRef, destStorage) <- files.fetchActiveStorage(dest.storage, dest.project, pc) + destFilesAttributes <- batchCopy.copyFiles(source, destStorage) + fileResources <- evalCreateCommands(pc, dest, destStorageRef, destStorage.tpe, destFilesAttributes) + } yield fileResources + }.span("copyFiles") + + private def evalCreateCommands( + pc: ProjectContext, + dest: CopyFileDestination, + destStorageRef: ResourceRef.Revision, + destStorageTpe: StorageType, + destFilesAttributes: NonEmptyList[FileAttributes] + )(implicit c: Caller): IO[NonEmptyList[FileResource]] = + destFilesAttributes.traverse { destFileAttributes => + for { + iri <- files.generateId(pc) + command = + CreateFile(iri, dest.project, destStorageRef, destStorageTpe, destFileAttributes, c.subject, dest.tag) + resource <- evalCreateCommand(files, command) + } yield resource + } + + private def evalCreateCommand(files: Files, command: CreateFile) = + files.eval(command).onError { e => + logger.error(e)(s"Failed storing file copy event, file must be manually deleted: $command") + } + } + +} diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileRejection.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileRejection.scala index b54d2efa1f..87f82ffa92 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileRejection.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileRejection.scala @@ -16,7 +16,6 @@ import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.JsonLdContext.keywords import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.encoder.JsonLdEncoder import ch.epfl.bluebrain.nexus.delta.sdk.marshalling.HttpResponseFields import ch.epfl.bluebrain.nexus.delta.sdk.marshalling.RdfRejectionHandler.all._ -import ch.epfl.bluebrain.nexus.delta.sdk.model.IdSegment import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContext import ch.epfl.bluebrain.nexus.delta.sdk.syntax.httpResponseFieldsSyntax import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef @@ -148,12 +147,6 @@ object FileRejection { s"Linking a file '$id' cannot be performed without a 'filename' or a 'path' that does not end with a filename." ) - /** - * Rejection returned when attempting to fetch a file and including both the target tag and revision. - */ - final case class InvalidFileLookup(id: IdSegment) - extends FileRejection(s"Only one of 'tag' and 'rev' can be used to lookup file '$id'.") - /** * Rejection returned when attempting to create/update a file with a Multipart/Form-Data payload that does not * contain a ''file'' fieldName diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskCopy.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskCopy.scala new file mode 100644 index 0000000000..4d6411b9c6 --- /dev/null +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskCopy.scala @@ -0,0 +1,56 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk + +import akka.http.scaladsl.model.Uri +import cats.data.NonEmptyList +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.kernel.utils.{CopyBetween, TransactionalFileCopier} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.DiskStorage +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.DiskStorageSaveFile.computeLocation +import fs2.io.file.Path + +import java.nio.file + +class DiskCopy(storage: DiskStorage, copier: TransactionalFileCopier) { + + def copyFiles(details: NonEmptyList[DiskCopyDetails]): IO[NonEmptyList[FileAttributes]] = + details + .traverse(mkCopyDetailsAndDestAttributes) + .flatMap { copyDetailsAndDestAttributes => + val copyDetails = copyDetailsAndDestAttributes.map(_._1) + val destAttrs = copyDetailsAndDestAttributes.map(_._2) + copier.copyAll(copyDetails).as(destAttrs) + } + + private def mkCopyDetailsAndDestAttributes(copyFile: DiskCopyDetails) = + for { + sourcePath <- absoluteDiskPathFromAttributes(copyFile.sourceAttributes) + (destPath, destRelativePath) <- computeDestLocation(copyFile) + destAttr = mkDestAttributes(copyFile, destPath, destRelativePath) + copyDetails <- absoluteDiskPathFromAttributes(destAttr).map { dest => + CopyBetween(Path.fromNioPath(sourcePath), Path.fromNioPath(dest)) + } + } yield (copyDetails, destAttr) + + private def computeDestLocation(copyFile: DiskCopyDetails): IO[(file.Path, file.Path)] = + computeLocation( + storage.project, + storage.value, + copyFile.destinationDesc.uuid, + copyFile.destinationDesc.filename + ) + + private def mkDestAttributes(copyFile: DiskCopyDetails, destPath: file.Path, destRelativePath: file.Path) = { + val dest = copyFile.destinationDesc + FileAttributes( + uuid = dest.uuid, + location = Uri(destPath.toUri.toString), + path = Uri.Path(destRelativePath.toString), + filename = dest.filename, + mediaType = copyFile.sourceAttributes.mediaType, + bytes = copyFile.sourceAttributes.bytes, + digest = copyFile.sourceAttributes.digest, + origin = copyFile.sourceAttributes.origin + ) + } +} diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskCopyDetails.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskCopyDetails.scala new file mode 100644 index 0000000000..182efab2ee --- /dev/null +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskCopyDetails.scala @@ -0,0 +1,10 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk + +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes, FileDescription} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.DiskStorage + +final case class DiskCopyDetails( + destStorage: DiskStorage, + destinationDesc: FileDescription, + sourceAttributes: FileAttributes +) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskCopy.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskCopy.scala new file mode 100644 index 0000000000..ef4f05fc94 --- /dev/null +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskCopy.scala @@ -0,0 +1,43 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote + +import akka.http.scaladsl.model.Uri +import cats.data.NonEmptyList +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.RemoteDiskStorage +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.SaveFile.intermediateFolders +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.RemoteDiskStorageClient + +class RemoteDiskCopy( + destStorage: RemoteDiskStorage, + client: RemoteDiskStorageClient +) { + + def copyFiles(copyDetails: NonEmptyList[RemoteDiskCopyDetails]): IO[NonEmptyList[FileAttributes]] = { + val paths = copyDetails.map { cd => + val destDesc = cd.destinationDesc + val destinationPath = + Uri.Path(intermediateFolders(destStorage.project, destDesc.uuid, destDesc.filename)) + val sourcePath = cd.sourceAttributes.path + (cd.sourceBucket, sourcePath, destinationPath) + } + + client.copyFile(destStorage.value.folder, paths)(destStorage.value.endpoint).map { destPaths => + copyDetails.zip(paths).zip(destPaths).map { case ((cd, x), destinationPath) => + val destDesc = cd.destinationDesc + val sourceAttr = cd.sourceAttributes + FileAttributes( + uuid = destDesc.uuid, + location = destinationPath, + path = x._3, + filename = destDesc.filename, + mediaType = destDesc.mediaType, + bytes = sourceAttr.bytes, + digest = sourceAttr.digest, + origin = sourceAttr.origin + ) + } + } + } + +} diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskCopyDetails.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskCopyDetails.scala new file mode 100644 index 0000000000..710a621d51 --- /dev/null +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskCopyDetails.scala @@ -0,0 +1,12 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote + +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes, FileDescription} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.RemoteDiskStorage +import ch.epfl.bluebrain.nexus.delta.sourcing.model.Label + +final case class RemoteDiskCopyDetails( + destStorage: RemoteDiskStorage, + destinationDesc: FileDescription, + sourceBucket: Label, + sourceAttributes: FileAttributes +) From 30668e0adf3bf6272583e6a1b2db857aa4e6bef6 Mon Sep 17 00:00:00 2001 From: dantb Date: Fri, 8 Dec 2023 11:37:42 +0100 Subject: [PATCH 09/18] Create BatchFilesRoutes, wire up with bulk file operations --- .../plugins/storage/StoragePluginModule.scala | 63 ++++++++- .../delta/plugins/storage/files/Files.scala | 18 +-- .../storage/files/batch/BatchCopy.scala | 22 ++-- .../storage/files/batch/BatchFiles.scala | 7 +- .../files/routes/BatchFilesRoutes.scala | 122 ++++++++++++++++++ .../storages/operations/disk/DiskCopy.scala | 81 ++++++------ .../operations/remote/RemoteDiskCopy.scala | 56 ++++---- ...CopyFileSpec.scala => CopyFilesSpec.scala} | 8 +- .../nexus/tests/kg/DiskStorageSpec.scala | 2 +- .../nexus/tests/kg/RemoteStorageSpec.scala | 2 +- 10 files changed, 281 insertions(+), 100 deletions(-) create mode 100644 delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesRoutes.scala rename tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/{CopyFileSpec.scala => CopyFilesSpec.scala} (95%) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala index c7add71b57..ad0e63ac52 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala @@ -7,14 +7,17 @@ import ch.epfl.bluebrain.nexus.delta.kernel.utils.{ClasspathResourceLoader, Tran import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.client.ElasticSearchClient import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.config.ElasticSearchViewsConfig import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.Files +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch.{BatchCopy, BatchFiles} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.contexts.{files => fileCtxId} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model._ -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.FilesRoutes +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.{BatchFilesRoutes, FilesRoutes} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.schemas.{files => filesSchemaId} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StoragesConfig.StorageTypeConfig import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.contexts.{storages => storageCtxId, storagesMetadata => storageMetaCtxId} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageAccess +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.DiskCopy +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.RemoteDiskCopy import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.RemoteDiskStorageClient import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.routes.StoragesRoutes import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.schemas.{storage => storagesSchemaId} @@ -186,6 +189,38 @@ class StoragePluginModule(priority: Int) extends ModuleDef { } } + make[TransactionalFileCopier].fromValue(TransactionalFileCopier.mk()) + + make[DiskCopy].from { copier: TransactionalFileCopier => DiskCopy.mk(copier)} + + make[RemoteDiskCopy].from { client: RemoteDiskStorageClient => RemoteDiskCopy.mk(client)} + + make[BatchCopy].from { + ( + files: Files, + storages: Storages, + storagesStatistics: StoragesStatistics, + diskCopy: DiskCopy, + remoteDiskCopy: RemoteDiskCopy, + uuidF: UUIDF + ) => + BatchCopy.mk(files, storages, storagesStatistics, diskCopy, remoteDiskCopy)(uuidF) + + } + + make[BatchFiles].from { + ( + fetchContext: FetchContext[ContextRejection], + files: Files, + batchCopy: BatchCopy, + ) => + BatchFiles.mk( + files, + fetchContext.mapRejection(FileRejection.ProjectContextRejection), + batchCopy + ) + } + make[FilesRoutes].from { ( cfg: StoragePluginConfig, @@ -210,6 +245,28 @@ class StoragePluginModule(priority: Int) extends ModuleDef { ) } + make[BatchFilesRoutes].from { + ( + cfg: StoragePluginConfig, + identities: Identities, + aclCheck: AclCheck, + batchFiles: BatchFiles, + schemeDirectives: DeltaSchemeDirectives, + indexingAction: AggregateIndexingAction, + shift: File.Shift, + baseUri: BaseUri, + cr: RemoteContextResolution@Id("aggregate"), + ordering: JsonKeyOrdering + ) => + val storageConfig = cfg.storages.storageTypeConfig + new BatchFilesRoutes(identities, aclCheck, batchFiles, schemeDirectives, indexingAction(_, _, _)(shift))( + baseUri, + storageConfig, + cr, + ordering + ) + } + make[File.Shift].from { (files: Files, base: BaseUri, storageTypeConfig: StorageTypeConfig) => File.shift(files)(base, storageTypeConfig) } @@ -284,4 +341,8 @@ class StoragePluginModule(priority: Int) extends ModuleDef { many[PriorityRoute].add { (fileRoutes: FilesRoutes) => PriorityRoute(priority, fileRoutes.routes, requiresStrictEntity = false) } + + many[PriorityRoute].add { (batchFileRoutes: BatchFilesRoutes) => + PriorityRoute(priority, batchFileRoutes.routes, requiresStrictEntity = false) + } } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala index e5493cf54b..caefb6e25b 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala @@ -100,7 +100,7 @@ final class Files( pc <- fetchContext.onCreate(projectRef) iri <- generateId(pc) _ <- test(CreateFile(iri, projectRef, testStorageRef, testStorageType, testAttributes, caller.subject, tag)) - (storageRef, storage) <- fetchActiveStorage(storageId, projectRef, pc) + (storageRef, storage) <- fetchAndValidateActiveStorage(storageId, projectRef, pc) attributes <- extractFileAttributes(iri, entity, storage) res <- eval(CreateFile(iri, projectRef, storageRef, storage.tpe, attributes, caller.subject, tag)) } yield res @@ -129,7 +129,7 @@ final class Files( for { (iri, pc) <- id.expandIri(fetchContext.onCreate) _ <- test(CreateFile(iri, id.project, testStorageRef, testStorageType, testAttributes, caller.subject, tag)) - (storageRef, storage) <- fetchActiveStorage(storageId, id.project, pc) + (storageRef, storage) <- fetchAndValidateActiveStorage(storageId, id.project, pc) attributes <- extractFileAttributes(iri, entity, storage) res <- eval(CreateFile(iri, id.project, storageRef, storage.tpe, attributes, caller.subject, tag)) } yield res @@ -257,7 +257,7 @@ final class Files( )(implicit c: Caller): IO[(ProjectContext, ResourceRef.Revision, Storage)] = for { pc <- fetchContext.onCreate(dest.project) - (destStorageRef, destStorage) <- fetchActiveStorage(dest.storage, dest.project, pc) + (destStorageRef, destStorage) <- fetchAndValidateActiveStorage(dest.storage, dest.project, pc) } yield (pc, destStorageRef, destStorage) private def validateStorageTypeForCopy(source: StorageType, destination: Storage): IO[Unit] = @@ -294,7 +294,7 @@ final class Files( for { (iri, pc) <- id.expandIri(fetchContext.onModify) _ <- test(UpdateFile(iri, id.project, testStorageRef, testStorageType, testAttributes, rev, caller.subject, tag)) - (storageRef, storage) <- fetchActiveStorage(storageId, id.project, pc) + (storageRef, storage) <- fetchAndValidateActiveStorage(storageId, id.project, pc) attributes <- extractFileAttributes(iri, entity, storage) res <- eval(UpdateFile(iri, id.project, storageRef, storage.tpe, attributes, rev, caller.subject, tag)) } yield res @@ -330,7 +330,7 @@ final class Files( for { (iri, pc) <- id.expandIri(fetchContext.onModify) _ <- test(UpdateFile(iri, id.project, testStorageRef, testStorageType, testAttributes, rev, caller.subject, tag)) - (storageRef, storage) <- fetchActiveStorage(storageId, id.project, pc) + (storageRef, storage) <- fetchAndValidateActiveStorage(storageId, id.project, pc) resolvedFilename <- IO.fromOption(filename.orElse(path.lastSegment))(InvalidFileLink(iri)) description <- FileDescription(resolvedFilename, mediaType) attributes <- linkFile(storage, path, description, iri) @@ -488,7 +488,7 @@ final class Files( )(implicit caller: Caller): IO[FileResource] = for { _ <- test(CreateFile(iri, ref, testStorageRef, testStorageType, testAttributes, caller.subject, tag)) - (storageRef, storage) <- fetchActiveStorage(storageId, ref, pc) + (storageRef, storage) <- fetchAndValidateActiveStorage(storageId, ref, pc) resolvedFilename <- IO.fromOption(filename.orElse(path.lastSegment))(InvalidFileLink(iri)) description <- FileDescription(resolvedFilename, mediaType) attributes <- linkFile(storage, path, description, iri) @@ -505,8 +505,8 @@ final class Files( private def test(cmd: FileCommand) = log.dryRun(cmd.project, cmd.id, cmd) - def fetchActiveStorage(storageIdOpt: Option[IdSegment], ref: ProjectRef, pc: ProjectContext)(implicit - caller: Caller + def fetchAndValidateActiveStorage(storageIdOpt: Option[IdSegment], ref: ProjectRef, pc: ProjectContext)(implicit + caller: Caller ): IO[(ResourceRef.Revision, Storage)] = storageIdOpt match { case Some(storageId) => @@ -550,7 +550,7 @@ final class Files( WrappedStorageRejection(s) } - def generateId(pc: ProjectContext)(implicit uuidF: UUIDF): IO[Iri] = + def generateId(pc: ProjectContext): IO[Iri] = uuidF().map(uuid => pc.base.iri / uuid.toString) /** diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopy.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopy.scala index 8105a50859..c7cb08b3bc 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopy.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopy.scala @@ -43,18 +43,18 @@ object BatchCopy { case s3: Storage.S3Storage => unsupported(s3.tpe) } - private def copyToRemoteStorage(source: CopyFileSource, remote: RemoteDiskStorage)(implicit c: Caller) = + private def copyToRemoteStorage(source: CopyFileSource, dest: RemoteDiskStorage)(implicit c: Caller) = for { - remoteCopyDetails <- source.files.traverse(fetchRemoteCopyDetails(remote, _)) - _ <- validateSpaceOnStorage(remote, remoteCopyDetails.map(_.sourceAttributes.bytes)) - attributes <- remoteDiskCopy.copyFiles(remoteCopyDetails) + remoteCopyDetails <- source.files.traverse(fetchRemoteCopyDetails(dest, _)) + _ <- validateSpaceOnStorage(dest, remoteCopyDetails.map(_.sourceAttributes.bytes)) + attributes <- remoteDiskCopy.copyFiles(dest, remoteCopyDetails) } yield attributes - private def copyToDiskStorage(source: CopyFileSource, disk: DiskStorage)(implicit c: Caller) = + private def copyToDiskStorage(source: CopyFileSource, dest: DiskStorage)(implicit c: Caller) = for { - diskCopyDetails <- source.files.traverse(fetchDiskCopyDetails(disk, _)) - _ <- validateSpaceOnStorage(disk, diskCopyDetails.map(_.sourceAttributes.bytes)) - attributes <- diskCopy.copyFiles(diskCopyDetails) + diskCopyDetails <- source.files.traverse(fetchDiskCopyDetails(dest, _)) + _ <- validateSpaceOnStorage(dest, diskCopyDetails.map(_.sourceAttributes.bytes)) + attributes <- diskCopy.copyFiles(dest, diskCopyDetails) } yield attributes private def validateSpaceOnStorage(destStorage: Storage, sourcesBytes: NonEmptyList[Long]): IO[Unit] = for { @@ -67,7 +67,7 @@ object BatchCopy { private def fetchDiskCopyDetails(destStorage: DiskStorage, fileId: FileId)(implicit c: Caller) = for { - (file, sourceStorage) <- fetchSourceFileAndStorage(fileId) + (file, sourceStorage) <- fetchFileAndValidateStorage(fileId) destinationDesc <- FileDescription(file.attributes.filename, file.attributes.mediaType) _ <- validateDiskStorage(destStorage, sourceStorage) } yield DiskCopyDetails(destStorage, destinationDesc, file.attributes) @@ -80,7 +80,7 @@ object BatchCopy { private def fetchRemoteCopyDetails(destStorage: RemoteDiskStorage, fileId: FileId)(implicit c: Caller) = for { - (file, sourceStorage) <- fetchSourceFileAndStorage(fileId) + (file, sourceStorage) <- fetchFileAndValidateStorage(fileId) destinationDesc <- FileDescription(file.attributes.filename, file.attributes.mediaType) sourceBucket <- validateRemoteStorage(destStorage, sourceStorage) } yield RemoteDiskCopyDetails(destStorage, destinationDesc, sourceBucket, file.attributes) @@ -96,7 +96,7 @@ object BatchCopy { private def unsupported(tpe: StorageType) = IO.raiseError(CopyFileRejection.UnsupportedOperation(tpe)) - private def fetchSourceFileAndStorage(id: FileId)(implicit c: Caller) = + private def fetchFileAndValidateStorage(id: FileId)(implicit c: Caller) = for { file <- files.fetch(id) sourceStorage <- storages.fetch(file.value.storage, id.project) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFiles.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFiles.scala index 512186f062..4183a30ab5 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFiles.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFiles.scala @@ -4,7 +4,6 @@ import cats.data.NonEmptyList import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.kernel.Logger import ch.epfl.bluebrain.nexus.delta.kernel.kamon.KamonMetricComponent -import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.Files.entityType import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileCommand._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model._ @@ -25,9 +24,7 @@ trait BatchFiles { } object BatchFiles { - def mk(files: Files, fetchContext: FetchContext[FileRejection], batchCopy: BatchCopy)(implicit - uuidF: UUIDF - ): BatchFiles = new BatchFiles { + def mk(files: Files, fetchContext: FetchContext[FileRejection], batchCopy: BatchCopy): BatchFiles = new BatchFiles { private val logger = Logger[BatchFiles] @@ -38,7 +35,7 @@ object BatchFiles { ): IO[NonEmptyList[FileResource]] = { for { pc <- fetchContext.onCreate(dest.project) - (destStorageRef, destStorage) <- files.fetchActiveStorage(dest.storage, dest.project, pc) + (destStorageRef, destStorage) <- files.fetchAndValidateActiveStorage(dest.storage, dest.project, pc) destFilesAttributes <- batchCopy.copyFiles(source, destStorage) fileResources <- evalCreateCommands(pc, dest, destStorageRef, destStorage.tpe, destFilesAttributes) } yield fileResources diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesRoutes.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesRoutes.scala new file mode 100644 index 0000000000..2e42271b76 --- /dev/null +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesRoutes.scala @@ -0,0 +1,122 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes + +import akka.http.scaladsl.model.StatusCodes.Created +import akka.http.scaladsl.server._ +import cats.data.EitherT +import cats.effect.IO +import cats.syntax.all._ +import ch.epfl.bluebrain.nexus.delta.kernel.Logger +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch.BatchFiles +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection._ +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{CopyFileDestination, File, FileRejection} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.permissions.{read => Read} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{FileResource, contexts} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StoragesConfig.StorageTypeConfig +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.{ContextValue, RemoteContextResolution} +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.encoder.JsonLdEncoder +import ch.epfl.bluebrain.nexus.delta.rdf.utils.JsonKeyOrdering +import ch.epfl.bluebrain.nexus.delta.sdk._ +import ch.epfl.bluebrain.nexus.delta.sdk.acls.AclCheck +import ch.epfl.bluebrain.nexus.delta.sdk.circe.CirceUnmarshalling +import ch.epfl.bluebrain.nexus.delta.sdk.directives.DeltaDirectives._ +import ch.epfl.bluebrain.nexus.delta.sdk.directives.{AuthDirectives, DeltaSchemeDirectives} +import ch.epfl.bluebrain.nexus.delta.sdk.error.ServiceError.AuthorizationFailed +import ch.epfl.bluebrain.nexus.delta.sdk.identities.Identities +import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller +import ch.epfl.bluebrain.nexus.delta.sdk.jsonld.BulkOperationResults +import ch.epfl.bluebrain.nexus.delta.sdk.model.{BaseUri, IdSegment} +import kamon.instrumentation.akka.http.TracingDirectives.operationName + +/** + * The files routes + * + * @param identities + * the identity module + * @param aclCheck + * to check acls + * @param files + * the files module + * @param schemeDirectives + * directives related to orgs and projects + * @param index + * the indexing action on write operations + */ +final class BatchFilesRoutes( + identities: Identities, + aclCheck: AclCheck, + batchFiles: BatchFiles, + schemeDirectives: DeltaSchemeDirectives, + index: IndexingAction.Execute[File] +)(implicit + baseUri: BaseUri, + storageConfig: StorageTypeConfig, + cr: RemoteContextResolution, + ordering: JsonKeyOrdering +) extends AuthDirectives(identities, aclCheck) + with CirceUnmarshalling { self => + + private val logger = Logger[BatchFilesRoutes] + + import baseUri.prefixSegment + import schemeDirectives._ + + implicit val bulkOpJsonLdEnc: JsonLdEncoder[BulkOperationResults[FileResource]] = + BulkOperationResults.searchResultsJsonLdEncoder(ContextValue(contexts.files)) + + def routes: Route = + baseUriPrefix(baseUri.prefix) { + pathPrefix("bulk") { + pathPrefix("files") { + extractCaller { implicit caller => + resolveProjectRef.apply { projectRef => + (post & pathEndOrSingleSlash & parameter("storage".as[IdSegment].?) & indexingMode & tagParam) { + (storage, mode, tag) => + operationName(s"$prefixSegment/files/{org}/{project}") { + // Bulk create files by copying from another project + entity(as[CopyFileSource]) { c: CopyFileSource => + val copyTo = CopyFileDestination(projectRef, storage, tag) + emit(Created, copyFile(mode, c, copyTo)) + } + } + } + } + } + } + } + } + + private def copyFile(mode: IndexingMode, source: CopyFileSource, dest: CopyFileDestination)(implicit + caller: Caller + ): IO[Either[FileRejection, BulkOperationResults[FileResource]]] = + (for { + _ <- + EitherT.right(aclCheck.authorizeForOr(source.project, Read)(AuthorizationFailed(source.project.project, Read))) + results <- EitherT(batchFiles.copyFiles(source, dest).attemptNarrow[FileRejection]) + _ <- EitherT.right[FileRejection](results.traverse(index(dest.project, _, mode))) + _ <- EitherT.right[FileRejection](logger.info(s"Bulk file copy succeeded with results: $results")) + } yield BulkOperationResults(results.toList)) + .onError(e => + EitherT.right(logger.error(s"Bulk file copy operation failed for source $source and destination $dest with $e")) + ) + .value +} + +object BatchFilesRoutes { + + def apply( + config: StorageTypeConfig, + identities: Identities, + aclCheck: AclCheck, + batchFiles: BatchFiles, + schemeDirectives: DeltaSchemeDirectives, + index: IndexingAction.Execute[File] + )(implicit + baseUri: BaseUri, + cr: RemoteContextResolution, + ordering: JsonKeyOrdering + ): Route = { + implicit val storageTypeConfig: StorageTypeConfig = config + new BatchFilesRoutes(identities, aclCheck, batchFiles, schemeDirectives, index).routes + } + +} diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskCopy.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskCopy.scala index 4d6411b9c6..bca18a24db 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskCopy.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskCopy.scala @@ -11,46 +11,45 @@ import fs2.io.file.Path import java.nio.file -class DiskCopy(storage: DiskStorage, copier: TransactionalFileCopier) { - - def copyFiles(details: NonEmptyList[DiskCopyDetails]): IO[NonEmptyList[FileAttributes]] = - details - .traverse(mkCopyDetailsAndDestAttributes) - .flatMap { copyDetailsAndDestAttributes => - val copyDetails = copyDetailsAndDestAttributes.map(_._1) - val destAttrs = copyDetailsAndDestAttributes.map(_._2) - copier.copyAll(copyDetails).as(destAttrs) - } - - private def mkCopyDetailsAndDestAttributes(copyFile: DiskCopyDetails) = - for { - sourcePath <- absoluteDiskPathFromAttributes(copyFile.sourceAttributes) - (destPath, destRelativePath) <- computeDestLocation(copyFile) - destAttr = mkDestAttributes(copyFile, destPath, destRelativePath) - copyDetails <- absoluteDiskPathFromAttributes(destAttr).map { dest => - CopyBetween(Path.fromNioPath(sourcePath), Path.fromNioPath(dest)) - } - } yield (copyDetails, destAttr) - - private def computeDestLocation(copyFile: DiskCopyDetails): IO[(file.Path, file.Path)] = - computeLocation( - storage.project, - storage.value, - copyFile.destinationDesc.uuid, - copyFile.destinationDesc.filename - ) - - private def mkDestAttributes(copyFile: DiskCopyDetails, destPath: file.Path, destRelativePath: file.Path) = { - val dest = copyFile.destinationDesc - FileAttributes( - uuid = dest.uuid, - location = Uri(destPath.toUri.toString), - path = Uri.Path(destRelativePath.toString), - filename = dest.filename, - mediaType = copyFile.sourceAttributes.mediaType, - bytes = copyFile.sourceAttributes.bytes, - digest = copyFile.sourceAttributes.digest, - origin = copyFile.sourceAttributes.origin - ) +trait DiskCopy { + def copyFiles(destStorage: DiskStorage, details: NonEmptyList[DiskCopyDetails]): IO[NonEmptyList[FileAttributes]] +} + +object DiskCopy { + def mk(copier: TransactionalFileCopier): DiskCopy = new DiskCopy { + + def copyFiles(destStorage: DiskStorage, details: NonEmptyList[DiskCopyDetails]): IO[NonEmptyList[FileAttributes]] = + details + .traverse(mkCopyDetailsAndDestAttributes(destStorage, _)) + .flatMap { copyDetailsAndDestAttributes => + val copyDetails = copyDetailsAndDestAttributes.map(_._1) + val destAttrs = copyDetailsAndDestAttributes.map(_._2) + copier.copyAll(copyDetails).as(destAttrs) + } + + private def mkCopyDetailsAndDestAttributes(destStorage: DiskStorage, copyFile: DiskCopyDetails) = + for { + sourcePath <- absoluteDiskPathFromAttributes(copyFile.sourceAttributes) + (destPath, destRelativePath) <- computeDestLocation(destStorage, copyFile) + destAttr = mkDestAttributes(copyFile, destPath, destRelativePath) + copyDetails <- absoluteDiskPathFromAttributes(destAttr).map { dest => + CopyBetween(Path.fromNioPath(sourcePath), Path.fromNioPath(dest)) + } + } yield (copyDetails, destAttr) + + private def computeDestLocation(destStorage: DiskStorage, cd: DiskCopyDetails): IO[(file.Path, file.Path)] = + computeLocation(destStorage.project, destStorage.value, cd.destinationDesc.uuid, cd.destinationDesc.filename) + + private def mkDestAttributes(cd: DiskCopyDetails, destPath: file.Path, destRelativePath: file.Path) = + FileAttributes( + uuid = cd.destinationDesc.uuid, + location = Uri(destPath.toUri.toString), + path = Uri.Path(destRelativePath.toString), + filename = cd.destinationDesc.filename, + mediaType = cd.sourceAttributes.mediaType, + bytes = cd.sourceAttributes.bytes, + digest = cd.sourceAttributes.digest, + origin = cd.sourceAttributes.origin + ) } } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskCopy.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskCopy.scala index ef4f05fc94..aba856796a 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskCopy.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskCopy.scala @@ -8,36 +8,38 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.Remo import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.SaveFile.intermediateFolders import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.RemoteDiskStorageClient -class RemoteDiskCopy( - destStorage: RemoteDiskStorage, - client: RemoteDiskStorageClient -) { +trait RemoteDiskCopy { + def copyFiles(destStorage: RemoteDiskStorage, copyDetails: NonEmptyList[RemoteDiskCopyDetails]): IO[NonEmptyList[FileAttributes]] +} - def copyFiles(copyDetails: NonEmptyList[RemoteDiskCopyDetails]): IO[NonEmptyList[FileAttributes]] = { - val paths = copyDetails.map { cd => - val destDesc = cd.destinationDesc - val destinationPath = - Uri.Path(intermediateFolders(destStorage.project, destDesc.uuid, destDesc.filename)) - val sourcePath = cd.sourceAttributes.path - (cd.sourceBucket, sourcePath, destinationPath) - } +object RemoteDiskCopy { - client.copyFile(destStorage.value.folder, paths)(destStorage.value.endpoint).map { destPaths => - copyDetails.zip(paths).zip(destPaths).map { case ((cd, x), destinationPath) => - val destDesc = cd.destinationDesc - val sourceAttr = cd.sourceAttributes - FileAttributes( - uuid = destDesc.uuid, - location = destinationPath, - path = x._3, - filename = destDesc.filename, - mediaType = destDesc.mediaType, - bytes = sourceAttr.bytes, - digest = sourceAttr.digest, - origin = sourceAttr.origin - ) + def mk(client: RemoteDiskStorageClient): RemoteDiskCopy = new RemoteDiskCopy { + def copyFiles(destStorage: RemoteDiskStorage, copyDetails: NonEmptyList[RemoteDiskCopyDetails]): IO[NonEmptyList[FileAttributes]] = { + val paths = copyDetails.map { cd => + val destDesc = cd.destinationDesc + val destinationPath = + Uri.Path(intermediateFolders(destStorage.project, destDesc.uuid, destDesc.filename)) + val sourcePath = cd.sourceAttributes.path + (cd.sourceBucket, sourcePath, destinationPath) + } + + client.copyFile(destStorage.value.folder, paths)(destStorage.value.endpoint).map { destPaths => + copyDetails.zip(paths).zip(destPaths).map { case ((cd, x), destinationPath) => + val destDesc = cd.destinationDesc + val sourceAttr = cd.sourceAttributes + FileAttributes( + uuid = destDesc.uuid, + location = destinationPath, + path = x._3, + filename = destDesc.filename, + mediaType = destDesc.mediaType, + bytes = sourceAttr.bytes, + digest = sourceAttr.digest, + origin = sourceAttr.origin + ) + } } } } - } diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/CopyFileSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/CopyFilesSpec.scala similarity index 95% rename from tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/CopyFileSpec.scala rename to tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/CopyFilesSpec.scala index edca249505..6b11f04b24 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/CopyFileSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/CopyFilesSpec.scala @@ -8,12 +8,12 @@ import ch.epfl.bluebrain.nexus.delta.kernel.utils.UrlUtils import ch.epfl.bluebrain.nexus.tests.HttpClient._ import ch.epfl.bluebrain.nexus.tests.Identity.storages.Coyote import ch.epfl.bluebrain.nexus.tests.Optics -import ch.epfl.bluebrain.nexus.tests.kg.CopyFileSpec.{Response, StorageDetails} +import ch.epfl.bluebrain.nexus.tests.kg.CopyFilesSpec.{Response, StorageDetails} import io.circe.syntax.KeyOps import io.circe.{Decoder, DecodingFailure, Json, JsonObject} import org.scalatest.Assertion -trait CopyFileSpec { self: StorageSpec => +trait CopyFilesSpec { self: StorageSpec => "Copying multiple files" should { @@ -68,7 +68,7 @@ trait CopyFileSpec { self: StorageSpec => ): IO[Assertion] = { val destProjRef = destStorage.projRef val payload = mkPayload(sourceProjRef, sourceFiles) - val uri = s"/files/$destProjRef?storage=nxv:${destStorage.storageId}" + val uri = s"/bulk/files/$destProjRef?storage=nxv:${destStorage.storageId}" for { response <- deltaClient.postAndReturn[Response](uri, payload, Coyote) { (json, response) => @@ -112,7 +112,7 @@ trait CopyFileSpec { self: StorageSpec => givenANewProjectAndStorageInExistingOrg(genId())(test) } -object CopyFileSpec { +object CopyFilesSpec { final case class StorageDetails(projRef: String, storageId: String) final case class Response(ids: List[String]) diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/DiskStorageSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/DiskStorageSpec.scala index 887982c97e..4e9b626add 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/DiskStorageSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/DiskStorageSpec.scala @@ -8,7 +8,7 @@ import ch.epfl.bluebrain.nexus.tests.iam.types.Permission import io.circe.Json import org.scalatest.Assertion -class DiskStorageSpec extends StorageSpec with CopyFileSpec { +class DiskStorageSpec extends StorageSpec with CopyFilesSpec { override def storageName: String = "disk" diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/RemoteStorageSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/RemoteStorageSpec.scala index c6bec7aae0..966f06c015 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/RemoteStorageSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/RemoteStorageSpec.scala @@ -17,7 +17,7 @@ import org.scalatest.Assertion import scala.annotation.nowarn import scala.sys.process._ -class RemoteStorageSpec extends StorageSpec with CopyFileSpec { +class RemoteStorageSpec extends StorageSpec with CopyFilesSpec { override def storageName: String = "external" From 6a4576b039ba3d397f9c57425eff108ce81ca162 Mon Sep 17 00:00:00 2001 From: dantb Date: Sun, 10 Dec 2023 20:43:58 +0100 Subject: [PATCH 10/18] Unit test batch files routes --- .../plugins/storage/StoragePluginModule.scala | 54 ++-- .../delta/plugins/storage/files/Files.scala | 2 +- .../files/routes/BatchFilesRoutes.scala | 4 +- .../storage/files/routes/FilesRoutes.scala | 2 +- .../storages/operations/disk/DiskCopy.scala | 12 +- .../operations/remote/RemoteDiskCopy.scala | 61 +++-- .../remote/RemoteDiskStorageCopyFiles.scala | 2 +- .../client/RemoteDiskStorageClient.scala | 8 +- .../client/model/RemoteDiskCopyPaths.scala | 10 + .../files/file-bulk-copy-response.json | 1 - .../storage/files/routes/BatchFilesMock.scala | 54 ++++ .../files/routes/BatchFilesRoutesSpec.scala | 234 ++++++++++++++++++ .../nexus/delta/sdk/utils/RouteFixtures.scala | 3 +- 13 files changed, 380 insertions(+), 67 deletions(-) create mode 100644 delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/model/RemoteDiskCopyPaths.scala create mode 100644 delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesMock.scala create mode 100644 delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesRoutesSpec.scala diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala index ad0e63ac52..54c1c555fa 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala @@ -191,17 +191,17 @@ class StoragePluginModule(priority: Int) extends ModuleDef { make[TransactionalFileCopier].fromValue(TransactionalFileCopier.mk()) - make[DiskCopy].from { copier: TransactionalFileCopier => DiskCopy.mk(copier)} + make[DiskCopy].from { copier: TransactionalFileCopier => DiskCopy.mk(copier) } - make[RemoteDiskCopy].from { client: RemoteDiskStorageClient => RemoteDiskCopy.mk(client)} + make[RemoteDiskCopy].from { client: RemoteDiskStorageClient => RemoteDiskCopy.mk(client) } make[BatchCopy].from { ( - files: Files, - storages: Storages, - storagesStatistics: StoragesStatistics, - diskCopy: DiskCopy, - remoteDiskCopy: RemoteDiskCopy, + files: Files, + storages: Storages, + storagesStatistics: StoragesStatistics, + diskCopy: DiskCopy, + remoteDiskCopy: RemoteDiskCopy, uuidF: UUIDF ) => BatchCopy.mk(files, storages, storagesStatistics, diskCopy, remoteDiskCopy)(uuidF) @@ -209,17 +209,17 @@ class StoragePluginModule(priority: Int) extends ModuleDef { } make[BatchFiles].from { - ( + ( fetchContext: FetchContext[ContextRejection], - files: Files, - batchCopy: BatchCopy, - ) => - BatchFiles.mk( - files, - fetchContext.mapRejection(FileRejection.ProjectContextRejection), - batchCopy - ) - } + files: Files, + batchCopy: BatchCopy + ) => + BatchFiles.mk( + files, + fetchContext.mapRejection(FileRejection.ProjectContextRejection), + batchCopy + ) + } make[FilesRoutes].from { ( @@ -247,16 +247,16 @@ class StoragePluginModule(priority: Int) extends ModuleDef { make[BatchFilesRoutes].from { ( - cfg: StoragePluginConfig, - identities: Identities, - aclCheck: AclCheck, - batchFiles: BatchFiles, - schemeDirectives: DeltaSchemeDirectives, - indexingAction: AggregateIndexingAction, - shift: File.Shift, - baseUri: BaseUri, - cr: RemoteContextResolution@Id("aggregate"), - ordering: JsonKeyOrdering + cfg: StoragePluginConfig, + identities: Identities, + aclCheck: AclCheck, + batchFiles: BatchFiles, + schemeDirectives: DeltaSchemeDirectives, + indexingAction: AggregateIndexingAction, + shift: File.Shift, + baseUri: BaseUri, + cr: RemoteContextResolution @Id("aggregate"), + ordering: JsonKeyOrdering ) => val storageConfig = cfg.storages.storageTypeConfig new BatchFilesRoutes(identities, aclCheck, batchFiles, schemeDirectives, indexingAction(_, _, _)(shift))( diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala index caefb6e25b..cf51fa4d63 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala @@ -506,7 +506,7 @@ final class Files( private def test(cmd: FileCommand) = log.dryRun(cmd.project, cmd.id, cmd) def fetchAndValidateActiveStorage(storageIdOpt: Option[IdSegment], ref: ProjectRef, pc: ProjectContext)(implicit - caller: Caller + caller: Caller ): IO[(ResourceRef.Revision, Storage)] = storageIdOpt match { case Some(storageId) => diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesRoutes.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesRoutes.scala index 2e42271b76..2cef55baef 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesRoutes.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesRoutes.scala @@ -10,7 +10,7 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch.BatchFiles import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{CopyFileDestination, File, FileRejection} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.permissions.{read => Read} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{FileResource, contexts} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{contexts, FileResource} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StoragesConfig.StorageTypeConfig import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.{ContextValue, RemoteContextResolution} import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.encoder.JsonLdEncoder @@ -58,7 +58,7 @@ final class BatchFilesRoutes( private val logger = Logger[BatchFilesRoutes] import baseUri.prefixSegment - import schemeDirectives._ + import schemeDirectives.resolveProjectRef implicit val bulkOpJsonLdEnc: JsonLdEncoder[BulkOperationResults[FileResource]] = BulkOperationResults.searchResultsJsonLdEncoder(ContextValue(contexts.files)) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutes.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutes.scala index df5c6d351b..a408c83007 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutes.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutes.scala @@ -195,7 +195,7 @@ final class FilesRoutes( ) } }, - (pathPrefix("tags")) { + pathPrefix("tags") { operationName(s"$prefixSegment/files/{org}/{project}/{id}/tags") { concat( // Fetch a file tags diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskCopy.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskCopy.scala index bca18a24db..67dff2f52c 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskCopy.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskCopy.scala @@ -23,18 +23,18 @@ object DiskCopy { .traverse(mkCopyDetailsAndDestAttributes(destStorage, _)) .flatMap { copyDetailsAndDestAttributes => val copyDetails = copyDetailsAndDestAttributes.map(_._1) - val destAttrs = copyDetailsAndDestAttributes.map(_._2) + val destAttrs = copyDetailsAndDestAttributes.map(_._2) copier.copyAll(copyDetails).as(destAttrs) } private def mkCopyDetailsAndDestAttributes(destStorage: DiskStorage, copyFile: DiskCopyDetails) = for { - sourcePath <- absoluteDiskPathFromAttributes(copyFile.sourceAttributes) + sourcePath <- absoluteDiskPathFromAttributes(copyFile.sourceAttributes) (destPath, destRelativePath) <- computeDestLocation(destStorage, copyFile) - destAttr = mkDestAttributes(copyFile, destPath, destRelativePath) - copyDetails <- absoluteDiskPathFromAttributes(destAttr).map { dest => - CopyBetween(Path.fromNioPath(sourcePath), Path.fromNioPath(dest)) - } + destAttr = mkDestAttributes(copyFile, destPath, destRelativePath) + copyDetails <- absoluteDiskPathFromAttributes(destAttr).map { dest => + CopyBetween(Path.fromNioPath(sourcePath), Path.fromNioPath(dest)) + } } yield (copyDetails, destAttr) private def computeDestLocation(destStorage: DiskStorage, cd: DiskCopyDetails): IO[(file.Path, file.Path)] = diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskCopy.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskCopy.scala index aba856796a..f887b98322 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskCopy.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskCopy.scala @@ -1,45 +1,60 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote import akka.http.scaladsl.model.Uri +import akka.http.scaladsl.model.Uri.Path import cats.data.NonEmptyList import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.RemoteDiskStorage import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.SaveFile.intermediateFolders import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.RemoteDiskStorageClient +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.model.RemoteDiskCopyPaths trait RemoteDiskCopy { - def copyFiles(destStorage: RemoteDiskStorage, copyDetails: NonEmptyList[RemoteDiskCopyDetails]): IO[NonEmptyList[FileAttributes]] + def copyFiles( + destStorage: RemoteDiskStorage, + copyDetails: NonEmptyList[RemoteDiskCopyDetails] + ): IO[NonEmptyList[FileAttributes]] } object RemoteDiskCopy { def mk(client: RemoteDiskStorageClient): RemoteDiskCopy = new RemoteDiskCopy { - def copyFiles(destStorage: RemoteDiskStorage, copyDetails: NonEmptyList[RemoteDiskCopyDetails]): IO[NonEmptyList[FileAttributes]] = { - val paths = copyDetails.map { cd => - val destDesc = cd.destinationDesc - val destinationPath = - Uri.Path(intermediateFolders(destStorage.project, destDesc.uuid, destDesc.filename)) - val sourcePath = cd.sourceAttributes.path - (cd.sourceBucket, sourcePath, destinationPath) - } + def copyFiles( + destStorage: RemoteDiskStorage, + copyDetails: NonEmptyList[RemoteDiskCopyDetails] + ): IO[NonEmptyList[FileAttributes]] = { + + val paths = remoteDiskCopyPaths(destStorage, copyDetails) - client.copyFile(destStorage.value.folder, paths)(destStorage.value.endpoint).map { destPaths => - copyDetails.zip(paths).zip(destPaths).map { case ((cd, x), destinationPath) => - val destDesc = cd.destinationDesc - val sourceAttr = cd.sourceAttributes - FileAttributes( - uuid = destDesc.uuid, - location = destinationPath, - path = x._3, - filename = destDesc.filename, - mediaType = destDesc.mediaType, - bytes = sourceAttr.bytes, - digest = sourceAttr.digest, - origin = sourceAttr.origin - ) + client.copyFiles(destStorage.value.folder, paths)(destStorage.value.endpoint).map { destPaths => + copyDetails.zip(paths).zip(destPaths).map { case ((copyDetails, remoteCopyPaths), absoluteDestPath) => + mkDestAttributes(copyDetails, remoteCopyPaths.destPath, absoluteDestPath) } } } } + + private def mkDestAttributes(cd: RemoteDiskCopyDetails, relativeDestPath: Path, absoluteDestPath: Uri) = { + val destDesc = cd.destinationDesc + val sourceAttr = cd.sourceAttributes + FileAttributes( + uuid = destDesc.uuid, + location = absoluteDestPath, + path = relativeDestPath, + filename = destDesc.filename, + mediaType = destDesc.mediaType, + bytes = sourceAttr.bytes, + digest = sourceAttr.digest, + origin = sourceAttr.origin + ) + } + + private def remoteDiskCopyPaths(destStorage: RemoteDiskStorage, copyDetails: NonEmptyList[RemoteDiskCopyDetails]) = + copyDetails.map { cd => + val destDesc = cd.destinationDesc + val destinationPath = Uri.Path(intermediateFolders(destStorage.project, destDesc.uuid, destDesc.filename)) + val sourcePath = cd.sourceAttributes.path + RemoteDiskCopyPaths(cd.sourceBucket, sourcePath, destinationPath) + } } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageCopyFiles.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageCopyFiles.scala index 63a72bd0a3..0c51891c5b 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageCopyFiles.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageCopyFiles.scala @@ -33,7 +33,7 @@ class RemoteDiskStorageCopyFiles( maybePaths.flatMap { paths => logger.info(s"DTBDTB REMOTE doing copy with ${destStorage.value.folder} and $paths") >> - client.copyFile(destStorage.value.folder, paths)(destStorage.value.endpoint).flatMap { destPaths => + client.copyFiles(destStorage.value.folder, paths)(destStorage.value.endpoint).flatMap { destPaths => logger.info(s"DTBDTB REMOTE received destPaths ${destPaths}").as { copyDetails.zip(paths).zip(destPaths).map { case ((cd, x), destinationPath) => FileAttributes( diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/RemoteDiskStorageClient.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/RemoteDiskStorageClient.scala index bf7a9013a7..1eb2a9b84a 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/RemoteDiskStorageClient.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/RemoteDiskStorageClient.scala @@ -13,7 +13,7 @@ import cats.implicits._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.FetchFileRejection.UnexpectedFetchError import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.MoveFileRejection.UnexpectedMoveError import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.{FetchFileRejection, MoveFileRejection, SaveFileRejection} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.model.RemoteDiskStorageFileAttributes +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.model.{RemoteDiskCopyPaths, RemoteDiskStorageFileAttributes} import ch.epfl.bluebrain.nexus.delta.rdf.implicits.uriDecoder import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.JsonLdContext.keywords import ch.epfl.bluebrain.nexus.delta.sdk.AkkaSource @@ -187,13 +187,13 @@ final class RemoteDiskStorageClient(client: HttpClient, getAuthToken: AuthTokenP * @param destRelativePath * the destination relative path location inside the nexus folder */ - def copyFile( + def copyFiles( destBucket: Label, - files: NonEmptyList[(Label, Path, Path)] + files: NonEmptyList[RemoteDiskCopyPaths] )(implicit baseUri: BaseUri): IO[NonEmptyList[Uri]] = { getAuthToken(credentials).flatMap { authToken => val endpoint = baseUri.endpoint / "buckets" / destBucket.value / "files" - val payload = files.map { case (sourceBucket, source, dest) => + val payload = files.map { case RemoteDiskCopyPaths(sourceBucket, source, dest) => Json.obj("sourceBucket" := sourceBucket, "source" := source.toString(), "destination" := dest.toString()) }.asJson diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/model/RemoteDiskCopyPaths.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/model/RemoteDiskCopyPaths.scala new file mode 100644 index 0000000000..fdff1e945e --- /dev/null +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/model/RemoteDiskCopyPaths.scala @@ -0,0 +1,10 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.model + +import akka.http.scaladsl.model.Uri.Path +import ch.epfl.bluebrain.nexus.delta.sourcing.model.Label + +final case class RemoteDiskCopyPaths( + sourceBucket: Label, + sourcePath: Path, + destPath: Path +) diff --git a/delta/plugins/storage/src/test/resources/files/file-bulk-copy-response.json b/delta/plugins/storage/src/test/resources/files/file-bulk-copy-response.json index 8781053488..82e8e03eee 100644 --- a/delta/plugins/storage/src/test/resources/files/file-bulk-copy-response.json +++ b/delta/plugins/storage/src/test/resources/files/file-bulk-copy-response.json @@ -4,6 +4,5 @@ "https://bluebrain.github.io/nexus/contexts/metadata.json", "https://bluebrain.github.io/nexus/contexts/files.json" ], - "_total": {{total}}, "_results": {{results}} } \ No newline at end of file diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesMock.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesMock.scala new file mode 100644 index 0000000000..991f853bb4 --- /dev/null +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesMock.scala @@ -0,0 +1,54 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes + +import cats.data.NonEmptyList +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.FileResource +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch.BatchFiles +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{CopyFileDestination, FileId} +import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller +import ch.epfl.bluebrain.nexus.delta.sdk.model.IdSegment +import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.User +import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef +import ch.epfl.bluebrain.nexus.delta.sourcing.model.Tag.UserTag + +import scala.collection.mutable.ListBuffer + +object BatchFilesMock { + + def unimplemented: BatchFiles = withMockedCopyFiles((_, _) => _ => IO(???)) + + def withStubbedCopyFiles( + stubbed: NonEmptyList[FileResource], + buffer: ListBuffer[BatchFilesCopyFilesCalled] + ): BatchFiles = + withMockedCopyFiles((source, dest) => + c => IO(buffer.addOne(BatchFilesCopyFilesCalled(source, dest, c))).as(stubbed) + ) + + def withMockedCopyFiles( + copyFilesMock: (CopyFileSource, CopyFileDestination) => Caller => IO[NonEmptyList[FileResource]] + ): BatchFiles = new BatchFiles { + override def copyFiles(source: CopyFileSource, dest: CopyFileDestination)(implicit + c: Caller + ): IO[NonEmptyList[FileResource]] = + copyFilesMock(source, dest)(c) + } + + final case class BatchFilesCopyFilesCalled(source: CopyFileSource, dest: CopyFileDestination, caller: Caller) + + object BatchFilesCopyFilesCalled { + def fromTestData( + destProj: ProjectRef, + sourceProj: ProjectRef, + sourceFiles: NonEmptyList[FileId], + user: User, + destStorage: Option[IdSegment] = None, + destTag: Option[UserTag] = None + ): BatchFilesCopyFilesCalled = { + val expectedCopyFileSource = CopyFileSource(sourceProj, sourceFiles) + val expectedCopyFileDestination = CopyFileDestination(destProj, destStorage, destTag) + BatchFilesCopyFilesCalled(expectedCopyFileSource, expectedCopyFileDestination, Caller(user, Set(user))) + } + } + +} diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesRoutesSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesRoutesSpec.scala new file mode 100644 index 0000000000..6580477c75 --- /dev/null +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesRoutesSpec.scala @@ -0,0 +1,234 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes + +import akka.http.scaladsl.model.StatusCodes +import akka.http.scaladsl.model.headers.OAuth2BearerToken +import akka.http.scaladsl.server.Route +import cats.data.NonEmptyList +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch.BatchFiles +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileId +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.BatchFilesMock.BatchFilesCopyFilesCalled +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.BatchFilesRoutesSpec.BatchFilesRoutesGenerators +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{contexts => fileContexts, schemas, FileFixtures, FileGen, FileResource} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StorageFixtures +import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri +import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary +import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.nxv +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.{ContextValue, RemoteContextResolution} +import ch.epfl.bluebrain.nexus.delta.sdk.IndexingAction +import ch.epfl.bluebrain.nexus.delta.sdk.acls.model.AclAddress +import ch.epfl.bluebrain.nexus.delta.sdk.acls.{AclCheck, AclSimpleCheck} +import ch.epfl.bluebrain.nexus.delta.sdk.directives.DeltaSchemeDirectives +import ch.epfl.bluebrain.nexus.delta.sdk.generators.ProjectGen +import ch.epfl.bluebrain.nexus.delta.sdk.identities.IdentitiesDummy +import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller +import ch.epfl.bluebrain.nexus.delta.sdk.model.{IdSegment, IdSegmentRef} +import ch.epfl.bluebrain.nexus.delta.sdk.permissions.model.Permission +import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContextDummy +import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.{ApiMappings, Project, ProjectContext} +import ch.epfl.bluebrain.nexus.delta.sdk.utils.BaseRouteSpec +import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.User +import ch.epfl.bluebrain.nexus.delta.sourcing.model.Tag.UserTag +import ch.epfl.bluebrain.nexus.delta.sourcing.model.{Label, ProjectRef, ResourceRef} +import ch.epfl.bluebrain.nexus.testkit.Generators +import io.circe.Json +import io.circe.syntax.KeyOps +import org.scalatest.{Assertion, Assertions} + +import scala.collection.mutable.ListBuffer + +class BatchFilesRoutesSpec + extends BaseRouteSpec + with StorageFixtures + with FileFixtures + with BatchFilesRoutesGenerators { + + implicit override def rcr: RemoteContextResolution = + RemoteContextResolution.fixedIO( + fileContexts.files -> ContextValue.fromFile("contexts/files.json"), + Vocabulary.contexts.metadata -> ContextValue.fromFile("contexts/metadata.json"), + Vocabulary.contexts.bulkOperation -> ContextValue.fromFile("contexts/bulk-operation.json"), + Vocabulary.contexts.error -> ContextValue.fromFile("contexts/error.json") + ) + + "Batch copying files between projects" should { + + "succeed for source files looked up by latest" in { + val sourceProj = genProject() + val sourceFileIds = genFilesIdsInProject(sourceProj.ref) + testBulkCopySucceedsForStubbedFiles(sourceProj, sourceFileIds) + } + + "succeed for source files looked up by tag" in { + val sourceProj = genProject() + val sourceFileIds = NonEmptyList.of(genFileIdWithTag(sourceProj.ref), genFileIdWithTag(sourceProj.ref)) + testBulkCopySucceedsForStubbedFiles(sourceProj, sourceFileIds) + } + + "succeed for source files looked up by rev" in { + val sourceProj = genProject() + val sourceFileIds = NonEmptyList.of(genFileIdWithRev(sourceProj.ref), genFileIdWithRev(sourceProj.ref)) + testBulkCopySucceedsForStubbedFiles(sourceProj, sourceFileIds) + } + + "succeed with a specific destination storage" in { + val sourceProj = genProject() + val sourceFileIds = genFilesIdsInProject(sourceProj.ref) + val destStorageId = IdSegment(genString()) + testBulkCopySucceedsForStubbedFiles(sourceProj, sourceFileIds, destStorageId = Some(destStorageId)) + } + + "succeed with a user tag applied to destination files" in { + val sourceProj = genProject() + val sourceFileIds = genFilesIdsInProject(sourceProj.ref) + val destTag = UserTag.unsafe(genString()) + testBulkCopySucceedsForStubbedFiles(sourceProj, sourceFileIds, destTag = Some(destTag)) + } + + "be rejected for a user without read permission on the source project" in { + val (sourceProj, destProj, user) = (genProject(), genProject(), genUser(realm)) + val sourceFileIds = genFilesIdsInProject(sourceProj.ref) + + val route = mkRoute(BatchFilesMock.unimplemented, sourceProj, user, permissions = Set()) + val payload = BatchFilesRoutesSpec.mkBulkCopyPayload(sourceProj.ref, sourceFileIds) + + callBulkCopyEndpoint(route, destProj.ref, payload, user) { + response.shouldBeForbidden + } + } + + "be rejected if tag and rev are present simultaneously for a source file" in { + val (sourceProj, destProj, user) = (genProject(), genProject(), genUser(realm)) + + val route = mkRoute(BatchFilesMock.unimplemented, sourceProj, user, permissions = Set()) + val invalidFilePayload = BatchFilesRoutesSpec.mkSourceFilePayload(genString(), Some(3), Some(genString())) + val payload = Json.obj("sourceProjectRef" := sourceProj.ref, "files" := List(invalidFilePayload)) + + callBulkCopyEndpoint(route, destProj.ref, payload, user) { + response.status shouldBe StatusCodes.BadRequest + } + } + } + + def mkRoute( + batchFiles: BatchFiles, + proj: Project, + user: User, + permissions: Set[Permission] + ): Route = { + val aclCheck: AclCheck = AclSimpleCheck((user, AclAddress.fromProject(proj.ref), permissions)).accepted + // TODO this dependency does nothing because we lookup using labels instead of UUIds in the tests... + // Does this endpoint need resolution by UUId? Do users need it? + val groupDirectives = DeltaSchemeDirectives(FetchContextDummy(Map(proj.ref -> proj.context))) + val identities = IdentitiesDummy(Caller(user, Set(user))) + Route.seal(BatchFilesRoutes(config, identities, aclCheck, batchFiles, groupDirectives, IndexingAction.noop)) + } + + def callBulkCopyEndpoint( + route: Route, + destProj: ProjectRef, + payload: Json, + user: User, + destStorageId: Option[IdSegment] = None, + destTag: Option[UserTag] = None + )(assert: => Assertion): Assertion = { + val asUser = addCredentials(OAuth2BearerToken(user.subject)) + val destStorageIdParam = destStorageId.map(id => s"storage=${id.asString}") + val destTagParam = destTag.map(tag => s"tag=${tag.value}") + val params = (destStorageIdParam.toList ++ destTagParam.toList).mkString("&") + Post(s"/v1/bulk/files/$destProj?$params", payload.toEntity()) ~> asUser ~> route ~> check(assert) + } + + def testBulkCopySucceedsForStubbedFiles( + sourceProj: Project, + sourceFileIds: NonEmptyList[FileId], + destStorageId: Option[IdSegment] = None, + destTag: Option[UserTag] = None + ): Assertion = { + val (destProj, user) = (genProject(), genUser(realm)) + val sourceFileResources = sourceFileIds.map(genFileResource(_, destProj.context)) + val events = ListBuffer.empty[BatchFilesCopyFilesCalled] + val stubbedBatchFiles = BatchFilesMock.withStubbedCopyFiles(sourceFileResources, events) + + val route = mkRoute(stubbedBatchFiles, sourceProj, user, Set(files.permissions.read)) + val payload = BatchFilesRoutesSpec.mkBulkCopyPayload(sourceProj.ref, sourceFileIds) + + callBulkCopyEndpoint(route, destProj.ref, payload, user, destStorageId, destTag) { + response.status shouldBe StatusCodes.Created + val expectedBatchFilesCall = + BatchFilesCopyFilesCalled.fromTestData( + destProj.ref, + sourceProj.ref, + sourceFileIds, + user, + destStorageId, + destTag + ) + events.toList shouldBe List(expectedBatchFilesCall) + response.asJson shouldBe expectedBulkCopyJson(sourceFileResources) + } + } + + def expectedBulkCopyJson(stubbedResources: NonEmptyList[FileResource]): Json = + Json.obj( + "@context" := List(Vocabulary.contexts.bulkOperation, Vocabulary.contexts.metadata, fileContexts.files), + "_results" := stubbedResources.map(expectedBulkOperationFileResourceJson) + ) + + def expectedBulkOperationFileResourceJson(res: FileResource): Json = + FilesRoutesSpec + .fileMetadata( + res.value.project, + res.id, + res.value.attributes, + res.value.storage, + res.value.storageType, + res.rev, + res.deprecated, + res.createdBy, + res.updatedBy + ) + .accepted + .mapObject(_.remove("@context")) +} + +object BatchFilesRoutesSpec { + + trait BatchFilesRoutesGenerators { self: Generators with FileFixtures with Assertions => + def genProjectRef(): ProjectRef = ProjectRef.unsafe(genString(), genString()) + def genProject(): Project = { + val projRef = genProjectRef() + val apiMappings = ApiMappings("file" -> schemas.files) + ProjectGen.project(projRef.project.value, projRef.organization.value, base = nxv.base, mappings = apiMappings) + } + + def genUser(realmLabel: Label): User = User(genString(), realmLabel) + def genFilesIdsInProject(projRef: ProjectRef): NonEmptyList[FileId] = + NonEmptyList.of(genString(), genString()).map(id => FileId(id, projRef)) + def genFileIdWithRev(projRef: ProjectRef): FileId = FileId(genString(), 4, projRef) + def genFileIdWithTag(projRef: ProjectRef): FileId = FileId(genString(), UserTag.unsafe(genString()), projRef) + + def genFileResource(fileId: FileId, context: ProjectContext): FileResource = + FileGen.resourceFor( + fileId.id.value.toIri(context.apiMappings, context.base).getOrElse(fail(s"Bad file $fileId")), + fileId.project, + ResourceRef.Revision(Iri.unsafe(genString()), 1), + attributes(genString()) + ) + } + + def mkBulkCopyPayload(sourceProj: ProjectRef, sourceFileIds: NonEmptyList[FileId]): Json = + Json.obj("sourceProjectRef" := sourceProj.toString, "files" := mkSourceFilesPayload(sourceFileIds)) + + def mkSourceFilesPayload(sourceFileIds: NonEmptyList[FileId]): NonEmptyList[Json] = + sourceFileIds.map(id => mkSourceFilePayloadFromIdSegmentRef(id.id)) + + def mkSourceFilePayloadFromIdSegmentRef(id: IdSegmentRef): Json = id match { + case IdSegmentRef.Latest(value) => mkSourceFilePayload(value.asString, None, None) + case IdSegmentRef.Revision(value, rev) => mkSourceFilePayload(value.asString, Some(rev), None) + case IdSegmentRef.Tag(value, tag) => mkSourceFilePayload(value.asString, None, Some(tag.value)) + } + + def mkSourceFilePayload(id: String, rev: Option[Int], tag: Option[String]): Json = + Json.obj("sourceFileId" := id, "sourceRev" := rev, "sourceTag" := tag) +} diff --git a/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/utils/RouteFixtures.scala b/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/utils/RouteFixtures.scala index 057979d322..ccce575679 100644 --- a/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/utils/RouteFixtures.scala +++ b/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/utils/RouteFixtures.scala @@ -46,7 +46,8 @@ trait RouteFixtures { contexts.supervision -> ContextValue.fromFile("contexts/supervision.json"), contexts.tags -> ContextValue.fromFile("contexts/tags.json"), contexts.version -> ContextValue.fromFile("contexts/version.json"), - contexts.quotas -> ContextValue.fromFile("contexts/quotas.json") + contexts.quotas -> ContextValue.fromFile("contexts/quotas.json"), + contexts.bulkOperation -> ContextValue.fromFile("contexts/bulk-operation.json") ) implicit val ordering: JsonKeyOrdering = From cba45f89fa56fe47c20d1344fb35cc39fb535c2e Mon Sep 17 00:00:00 2001 From: dantb Date: Mon, 11 Dec 2023 14:25:48 +0100 Subject: [PATCH 11/18] Reduce dependency surface area for BatchFiles + unit test it --- .../plugins/archive/ArchiveDownloadSpec.scala | 3 +- .../plugins/archive/ArchiveRoutesSpec.scala | 3 +- .../plugins/storage/StoragePluginModule.scala | 15 ++- .../storage/files/FetchFileStorage.scala | 14 ++ .../delta/plugins/storage/files/Files.scala | 10 +- .../storage/files/batch/BatchFiles.scala | 24 +++- .../remote/RemoteDiskStorageCopyFiles.scala | 5 +- .../storage/files/BatchFilesSpec.scala | 100 ++++++++++++++ .../plugins/storage/files/FileFixtures.scala | 8 +- .../delta/plugins/storage/files/FileGen.scala | 56 -------- .../plugins/storage/files/FilesSpec.scala | 1 + .../plugins/storage/files/FilesStmSpec.scala | 1 + .../storage/files/generators/FileGen.scala | 124 ++++++++++++++++++ .../{routes => mocks}/BatchFilesMock.scala | 3 +- .../files/routes/BatchFilesRoutesSpec.scala | 47 ++----- 15 files changed, 295 insertions(+), 119 deletions(-) create mode 100644 delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FetchFileStorage.scala create mode 100644 delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/BatchFilesSpec.scala delete mode 100644 delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FileGen.scala create mode 100644 delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/generators/FileGen.scala rename delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/{routes => mocks}/BatchFilesMock.scala (93%) diff --git a/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveDownloadSpec.scala b/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveDownloadSpec.scala index 567270e4c7..c9ffed6f23 100644 --- a/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveDownloadSpec.scala +++ b/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveDownloadSpec.scala @@ -14,10 +14,11 @@ import ch.epfl.bluebrain.nexus.delta.plugins.archive.model.ArchiveReference.{Fil import ch.epfl.bluebrain.nexus.delta.plugins.archive.model.ArchiveRejection.{InvalidFileSelf, ResourceNotFound} import ch.epfl.bluebrain.nexus.delta.plugins.archive.model.{ArchiveRejection, ArchiveValue} import ch.epfl.bluebrain.nexus.delta.plugins.storage.RemoteContextResolutionFixture +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.generators.FileGen import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes.FileAttributesOrigin.Client import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection.FileNotFound import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{Digest, FileAttributes} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{schemas, FileGen} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.schemas import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StorageFixtures import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.AbsolutePath import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri diff --git a/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveRoutesSpec.scala b/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveRoutesSpec.scala index 6398723743..91a3ab3e83 100644 --- a/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveRoutesSpec.scala +++ b/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveRoutesSpec.scala @@ -14,12 +14,13 @@ import ch.epfl.bluebrain.nexus.delta.kernel.utils.{StatefulUUIDF, UUIDF} import ch.epfl.bluebrain.nexus.delta.plugins.archive.FileSelf.ParsingError.InvalidPath import ch.epfl.bluebrain.nexus.delta.plugins.archive.model.ArchiveRejection.ProjectContextRejection import ch.epfl.bluebrain.nexus.delta.plugins.archive.routes.ArchiveRoutes +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.generators.FileGen import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest.ComputedDigest import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes.FileAttributesOrigin.Client import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection.FileNotFound import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{File, FileAttributes} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.FilesRoutesSpec -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{schemas, FileGen} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.schemas import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StorageFixtures import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.DigestAlgorithm import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala index 54c1c555fa..64f0d24a32 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala @@ -7,6 +7,7 @@ import ch.epfl.bluebrain.nexus.delta.kernel.utils.{ClasspathResourceLoader, Tran import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.client.ElasticSearchClient import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.config.ElasticSearchViewsConfig import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.Files +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.Files.FilesLog import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch.{BatchCopy, BatchFiles} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.contexts.{files => fileCtxId} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model._ @@ -43,7 +44,7 @@ import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContext.ContextRejection import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.ApiMappings import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverContextResolution import ch.epfl.bluebrain.nexus.delta.sdk.sse.SseEncoder -import ch.epfl.bluebrain.nexus.delta.sourcing.Transactors +import ch.epfl.bluebrain.nexus.delta.sourcing.{ScopedEventLog, Transactors} import ch.epfl.bluebrain.nexus.delta.sourcing.model.Label import ch.epfl.bluebrain.nexus.delta.sourcing.stream.Supervisor import com.typesafe.config.Config @@ -150,6 +151,11 @@ class StoragePluginModule(priority: Int) extends ModuleDef { many[ResourceShift[_, _, _]].ref[Storage.Shift] + // TODO refactor Files to depend on this rather than constructing it + make[FilesLog].from { (cfg: StoragePluginConfig, xas: Transactors, clock: Clock[IO]) => + ScopedEventLog(Files.definition(clock), cfg.files.eventLog, xas) + } + make[Files] .fromEffect { ( @@ -212,13 +218,16 @@ class StoragePluginModule(priority: Int) extends ModuleDef { ( fetchContext: FetchContext[ContextRejection], files: Files, - batchCopy: BatchCopy + filesLog: FilesLog, + batchCopy: BatchCopy, + uuidF: UUIDF ) => BatchFiles.mk( files, fetchContext.mapRejection(FileRejection.ProjectContextRejection), + FilesLog.eval(filesLog), batchCopy - ) + )(uuidF) } make[FilesRoutes].from { diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FetchFileStorage.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FetchFileStorage.scala new file mode 100644 index 0000000000..d97b030b03 --- /dev/null +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FetchFileStorage.scala @@ -0,0 +1,14 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files + +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage +import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller +import ch.epfl.bluebrain.nexus.delta.sdk.model._ +import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.ProjectContext +import ch.epfl.bluebrain.nexus.delta.sourcing.model.{ProjectRef, ResourceRef} + +trait FetchFileStorage { + def fetchAndValidateActiveStorage(storageIdOpt: Option[IdSegment], ref: ProjectRef, pc: ProjectContext)(implicit + caller: Caller + ): IO[(ResourceRef.Revision, Storage)] +} diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala index cf51fa4d63..d996607d5b 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala @@ -68,7 +68,7 @@ final class Files( )(implicit uuidF: UUIDF, system: ClassicActorSystem -) { +) extends FetchFileStorage { implicit private val kamonComponent: KamonMetricComponent = KamonMetricComponent(entityType.value) @@ -500,8 +500,7 @@ final class Files( .apply(path, desc) .adaptError { case e: StorageFileRejection => LinkRejection(fileId, storage.id, e) } - def eval(cmd: FileCommand): IO[FileResource] = - log.evaluate(cmd.project, cmd.id, cmd).map(_._2.toResource) + private def eval(cmd: FileCommand): IO[FileResource] = FilesLog.eval(log)(cmd) private def test(cmd: FileCommand) = log.dryRun(cmd.project, cmd.id, cmd) @@ -669,6 +668,11 @@ object Files { type FilesLog = ScopedEventLog[Iri, FileState, FileCommand, FileEvent, FileRejection] + object FilesLog { + def eval(log: FilesLog)(cmd: FileCommand): IO[FileResource] = + log.evaluate(cmd.project, cmd.id, cmd).map(_._2.toResource) + } + private[files] def next( state: Option[FileState], event: FileEvent diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFiles.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFiles.scala index 4183a30ab5..a8f2628fd0 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFiles.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFiles.scala @@ -4,12 +4,14 @@ import cats.data.NonEmptyList import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.kernel.Logger import ch.epfl.bluebrain.nexus.delta.kernel.kamon.KamonMetricComponent +import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.Files.entityType import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileCommand._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.CopyFileSource -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{FileResource, Files} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{FetchFileStorage, FileResource} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageType +import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller import ch.epfl.bluebrain.nexus.delta.sdk.implicits._ import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContext @@ -24,7 +26,12 @@ trait BatchFiles { } object BatchFiles { - def mk(files: Files, fetchContext: FetchContext[FileRejection], batchCopy: BatchCopy): BatchFiles = new BatchFiles { + def mk( + fetchFileStorage: FetchFileStorage, + fetchContext: FetchContext[FileRejection], + evalFileCommand: CreateFile => IO[FileResource], + batchCopy: BatchCopy + )(implicit uuidF: UUIDF): BatchFiles = new BatchFiles { private val logger = Logger[BatchFiles] @@ -35,7 +42,7 @@ object BatchFiles { ): IO[NonEmptyList[FileResource]] = { for { pc <- fetchContext.onCreate(dest.project) - (destStorageRef, destStorage) <- files.fetchAndValidateActiveStorage(dest.storage, dest.project, pc) + (destStorageRef, destStorage) <- fetchFileStorage.fetchAndValidateActiveStorage(dest.storage, dest.project, pc) destFilesAttributes <- batchCopy.copyFiles(source, destStorage) fileResources <- evalCreateCommands(pc, dest, destStorageRef, destStorage.tpe, destFilesAttributes) } yield fileResources @@ -50,15 +57,18 @@ object BatchFiles { )(implicit c: Caller): IO[NonEmptyList[FileResource]] = destFilesAttributes.traverse { destFileAttributes => for { - iri <- files.generateId(pc) + iri <- generateId(pc) command = CreateFile(iri, dest.project, destStorageRef, destStorageTpe, destFileAttributes, c.subject, dest.tag) - resource <- evalCreateCommand(files, command) + resource <- evalCreateCommand(command) } yield resource } - private def evalCreateCommand(files: Files, command: CreateFile) = - files.eval(command).onError { e => + def generateId(pc: ProjectContext): IO[Iri] = + uuidF().map(uuid => pc.base.iri / uuid.toString) + + private def evalCreateCommand(command: CreateFile) = + evalFileCommand(command).onError { e => logger.error(e)(s"Failed storing file copy event, file must be manually deleted: $command") } } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageCopyFiles.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageCopyFiles.scala index 0c51891c5b..442b715caf 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageCopyFiles.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageCopyFiles.scala @@ -10,6 +10,7 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageValue import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.CopyFiles import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.SaveFile.intermediateFolders import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.RemoteDiskStorageClient +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.model.RemoteDiskCopyPaths class RemoteDiskStorageCopyFiles( destStorage: RemoteDiskStorage, @@ -28,7 +29,7 @@ class RemoteDiskStorageCopyFiles( case remote: StorageValue.RemoteDiskStorageValue => IO(remote.folder) case other => IO.raiseError(new Exception(s"Invalid storage type for remote copy: $other")) } - thingy.map(sourceBucket => (sourceBucket, sourcePath, destinationPath)) + thingy.map(sourceBucket => RemoteDiskCopyPaths(sourceBucket, sourcePath, destinationPath)) } maybePaths.flatMap { paths => @@ -39,7 +40,7 @@ class RemoteDiskStorageCopyFiles( FileAttributes( uuid = cd.destinationDesc.uuid, location = destinationPath, - path = x._3, + path = x.destPath, filename = cd.destinationDesc.filename, mediaType = cd.destinationDesc.mediaType, bytes = cd.sourceAttributes.bytes, diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/BatchFilesSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/BatchFilesSpec.scala new file mode 100644 index 0000000000..5a68019d00 --- /dev/null +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/BatchFilesSpec.scala @@ -0,0 +1,100 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files + +import cats.data.NonEmptyList +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.BatchFilesSpec._ +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch.{BatchCopy, BatchFiles} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.generators.FileGen +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileCommand.CreateFile +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes, FileCommand, FileRejection} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.CopyFileSource +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StorageFixtures +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage +import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller +import ch.epfl.bluebrain.nexus.delta.sdk.model.IdSegment +import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.{Project, ProjectContext} +import ch.epfl.bluebrain.nexus.delta.sdk.projects.{FetchContext, FetchContextDummy} +import ch.epfl.bluebrain.nexus.delta.sourcing.model.{ProjectRef, ResourceRef} +import ch.epfl.bluebrain.nexus.testkit.Generators +import ch.epfl.bluebrain.nexus.testkit.mu.NexusSuite + +import java.util.UUID +import scala.collection.mutable.ListBuffer + +class BatchFilesSpec extends NexusSuite with StorageFixtures with Generators with FileFixtures with FileGen { + + test("batch copying files should fetch storage, perform copy and evaluate create file commands") { + val events = ListBuffer.empty[Event] + val destProj: Project = genProject() + val (destStorageRef, destStorage) = (genRevision(), genStorage(destProj.ref, diskVal)) + val fetchFileStorage = mockFetchFileStorage(destStorageRef, destStorage.storage, events) + val stubbedDestAttributes = genAttributes() + val batchCopy = mockBatchCopy(events, stubbedDestAttributes) + val destFileUUId = UUID.randomUUID() // Not testing UUID generation, same for all of them + + val batchFiles: BatchFiles = mkBatchFiles(events, destProj, destFileUUId, fetchFileStorage, batchCopy) + implicit val c: Caller = Caller(genUser(), Set()) + val (source, destination) = (genCopyFileSource(), genCopyFileDestination(destProj.ref, destStorage.storage)) + val obtained = batchFiles.copyFiles(source, destination).accepted + + val expectedFileIri = destProj.base.iri / destFileUUId.toString + val expectedCmds = stubbedDestAttributes.map( + CreateFile(expectedFileIri, destProj.ref, destStorageRef, destStorage.value.tpe, _, c.subject, destination.tag) + ) + + // resources returned are based on file command evaluation + assertEquals(obtained, expectedCmds.map(genFileResourceFromCmd)) + + val expectedActiveStorageFetched = ActiveStorageFetched(destination.storage, destProj.ref, destProj.context, c) + val expectedBatchCopyCalled = BatchCopyCalled(source, destStorage.storage, c) + val expectedCommandsEvaluated = expectedCmds.toList.map(FileCommandEvaluated) + val expectedEvents = List(expectedActiveStorageFetched, expectedBatchCopyCalled) ++ expectedCommandsEvaluated + assertEquals(events.toList, expectedEvents) + } + + def mockFetchFileStorage( + storageRef: ResourceRef.Revision, + storage: Storage, + events: ListBuffer[Event] + ): FetchFileStorage = new FetchFileStorage { + override def fetchAndValidateActiveStorage(storageIdOpt: Option[IdSegment], ref: ProjectRef, pc: ProjectContext)( + implicit caller: Caller + ): IO[(ResourceRef.Revision, Storage)] = + IO(events.addOne(ActiveStorageFetched(storageIdOpt, ref, pc, caller))).as(storageRef -> storage) + } + + def mockBatchCopy(events: ListBuffer[Event], stubbedAttr: NonEmptyList[FileAttributes]) = new BatchCopy { + override def copyFiles(source: CopyFileSource, destStorage: Storage)(implicit + c: Caller + ): IO[NonEmptyList[FileAttributes]] = + IO(events.addOne(BatchCopyCalled(source, destStorage, c))).as(stubbedAttr) + } + + def mkBatchFiles( + events: ListBuffer[Event], + proj: Project, + fixedUuid: UUID, + fetchFileStorage: FetchFileStorage, + batchCopy: BatchCopy + ): BatchFiles = { + implicit val uuidF: UUIDF = UUIDF.fixed(fixedUuid) + val evalFileCmd: CreateFile => IO[FileResource] = cmd => + IO(events.addOne(FileCommandEvaluated(cmd))).as(genFileResourceFromCmd(cmd)) + val fetchContext: FetchContext[FileRejection] = + FetchContextDummy(Map(proj.ref -> proj.context)).mapRejection(FileRejection.ProjectContextRejection) + BatchFiles.mk(fetchFileStorage, fetchContext, evalFileCmd, batchCopy) + } +} + +object BatchFilesSpec { + sealed trait Event + final case class ActiveStorageFetched( + storageIdOpt: Option[IdSegment], + ref: ProjectRef, + pc: ProjectContext, + caller: Caller + ) extends Event + final case class BatchCopyCalled(source: CopyFileSource, destStorage: Storage, caller: Caller) extends Event + final case class FileCommandEvaluated(cmd: FileCommand) extends Event +} diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FileFixtures.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FileFixtures.scala index 35043e76c8..cf34480c00 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FileFixtures.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FileFixtures.scala @@ -13,15 +13,11 @@ import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.nxv import ch.epfl.bluebrain.nexus.delta.sdk.generators.ProjectGen import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.ApiMappings import ch.epfl.bluebrain.nexus.delta.sourcing.model.{Label, ProjectRef} -import ch.epfl.bluebrain.nexus.testkit.scalatest.EitherValues -import org.scalatest.Suite import java.nio.file.{Files => JavaFiles} import java.util.{Base64, UUID} -trait FileFixtures extends EitherValues { - - self: Suite => +trait FileFixtures { val uuid = UUID.fromString("8249ba90-7cc6-4de5-93a1-802c04200dcc") val uuid2 = UUID.fromString("12345678-7cc6-4de5-93a1-802c04200dcc") @@ -49,7 +45,7 @@ trait FileFixtures extends EitherValues { val generatedId2 = project.base.iri / uuid2.toString val content = "file content" - val path = AbsolutePath(JavaFiles.createTempDirectory("files")).rightValue + val path = AbsolutePath(JavaFiles.createTempDirectory("files")).fold(e => throw new Exception(e), identity) val digest = ComputedDigest(DigestAlgorithm.default, "e0ac3601005dfa1864f5392aabaf7d898b1b5bab854f1acb4491bcd806b76b0c") diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FileGen.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FileGen.scala deleted file mode 100644 index c6b63b13cc..0000000000 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FileGen.scala +++ /dev/null @@ -1,56 +0,0 @@ -package ch.epfl.bluebrain.nexus.delta.plugins.storage.files - -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes, FileState} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageType -import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri -import ch.epfl.bluebrain.nexus.delta.sdk.model.Tags -import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.{Anonymous, Subject} -import ch.epfl.bluebrain.nexus.delta.sourcing.model.{ProjectRef, ResourceRef} - -import java.time.Instant - -object FileGen { - - def state( - id: Iri, - project: ProjectRef, - storage: ResourceRef.Revision, - attributes: FileAttributes, - storageType: StorageType = StorageType.DiskStorage, - rev: Int = 1, - deprecated: Boolean = false, - tags: Tags = Tags.empty, - createdBy: Subject = Anonymous, - updatedBy: Subject = Anonymous - ): FileState = { - FileState( - id, - project, - storage, - storageType, - attributes, - tags, - rev, - deprecated, - Instant.EPOCH, - createdBy, - Instant.EPOCH, - updatedBy - ) - } - - def resourceFor( - id: Iri, - project: ProjectRef, - storage: ResourceRef.Revision, - attributes: FileAttributes, - storageType: StorageType = StorageType.DiskStorage, - rev: Int = 1, - deprecated: Boolean = false, - tags: Tags = Tags.empty, - createdBy: Subject = Anonymous, - updatedBy: Subject = Anonymous - ): FileResource = - state(id, project, storage, attributes, storageType, rev, deprecated, tags, createdBy, updatedBy).toResource - -} diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala index 436007f0f1..bc5d9d223b 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala @@ -11,6 +11,7 @@ import cats.effect.unsafe.implicits.global import ch.epfl.bluebrain.nexus.delta.kernel.http.MediaTypeDetectorConfig import ch.epfl.bluebrain.nexus.delta.kernel.utils.TransactionalFileCopier import ch.epfl.bluebrain.nexus.delta.plugins.storage.RemoteContextResolutionFixture +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.generators.FileGen import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest.NotComputedDigest import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes.FileAttributesOrigin.Storage import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection._ diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesStmSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesStmSpec.scala index 0f6d1599b4..49b8d03841 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesStmSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesStmSpec.scala @@ -2,6 +2,7 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.files import akka.http.scaladsl.model.{ContentTypes, Uri} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.Files.{evaluate, next} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.generators.FileGen import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest.{ComputedDigest, NotComputedDigest} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes.FileAttributesOrigin.Client import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileCommand._ diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/generators/FileGen.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/generators/FileGen.scala new file mode 100644 index 0000000000..b52369f132 --- /dev/null +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/generators/FileGen.scala @@ -0,0 +1,124 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.generators + +import cats.data.NonEmptyList +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileCommand.CreateFile +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{CopyFileDestination, FileAttributes, FileId, FileState} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.CopyFileSource +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{schemas, FileFixtures, FileResource} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StorageGen +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{Storage, StorageState, StorageType, StorageValue} +import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri +import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.nxv +import ch.epfl.bluebrain.nexus.delta.sdk.generators.ProjectGen +import ch.epfl.bluebrain.nexus.delta.sdk.model.{IdSegment, Tags} +import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.{ApiMappings, Project, ProjectContext} +import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.{Anonymous, Subject, User} +import ch.epfl.bluebrain.nexus.delta.sourcing.model.Tag.UserTag +import ch.epfl.bluebrain.nexus.delta.sourcing.model.{Label, ProjectRef, ResourceRef} +import ch.epfl.bluebrain.nexus.testkit.Generators + +import java.time.Instant +import scala.util.Random + +trait FileGen { self: Generators with FileFixtures => + def genProjectRef(): ProjectRef = ProjectRef.unsafe(genString(), genString()) + + def genProject(): Project = { + val projRef = genProjectRef() + val apiMappings = ApiMappings("file" -> schemas.files) + ProjectGen.project(projRef.project.value, projRef.organization.value, base = nxv.base, mappings = apiMappings) + } + + def genUser(realmLabel: Label): User = User(genString(), realmLabel) + def genUser(): User = User(genString(), Label.unsafe(genString())) + + def genFilesIdsInProject(projRef: ProjectRef): NonEmptyList[FileId] = + NonEmptyList.of(genString(), genString()).map(id => FileId(id, projRef)) + + def genFileIdWithRev(projRef: ProjectRef): FileId = FileId(genString(), 4, projRef) + + def genFileIdWithTag(projRef: ProjectRef): FileId = FileId(genString(), UserTag.unsafe(genString()), projRef) + + def genAttributes(): NonEmptyList[FileAttributes] = { + val proj = genProject() + genFilesIdsInProject(proj.ref).map(genFileResource(_, proj.context)).map(_.value.attributes) + } + + def genCopyFileSource(): CopyFileSource = genCopyFileSource(genProjectRef()) + def genCopyFileSource(proj: ProjectRef) = CopyFileSource(proj, genFilesIdsInProject(proj)) + def genCopyFileDestination(proj: ProjectRef, storage: Storage): CopyFileDestination = + CopyFileDestination(proj, genOption(IdSegment(storage.id.toString)), genOption(genUserTag)) + def genUserTag: UserTag = UserTag.unsafe(genString()) + def genOption[A](genA: => A): Option[A] = if (Random.nextInt(2) % 2 == 0) Some(genA) else None + def genFileResource(fileId: FileId, context: ProjectContext): FileResource = + genFileResourceWithIri( + fileId.id.value.toIri(context.apiMappings, context.base).getOrElse(throw new Exception(s"Bad file $fileId")), + fileId.project, + genRevision(), + attributes(genString()) + ) + + def genFileResourceWithIri( + iri: Iri, + projRef: ProjectRef, + storageRef: ResourceRef.Revision, + attr: FileAttributes + ): FileResource = + FileGen.resourceFor(iri, projRef, storageRef, attr) + + def genFileResourceFromCmd(cmd: CreateFile): FileResource = + genFileResourceWithIri(cmd.id, cmd.project, cmd.storage, cmd.attributes) + def genIri(): Iri = Iri.unsafe(genString()) + def genStorage(proj: ProjectRef, storageValue: StorageValue): StorageState = + StorageGen.storageState(genIri(), proj, storageValue) + + def genRevision(): ResourceRef.Revision = + ResourceRef.Revision(genIri(), genPosInt()) + def genPosInt(): Int = Random.nextInt(Int.MaxValue) +} + +object FileGen { + + def state( + id: Iri, + project: ProjectRef, + storage: ResourceRef.Revision, + attributes: FileAttributes, + storageType: StorageType = StorageType.DiskStorage, + rev: Int = 1, + deprecated: Boolean = false, + tags: Tags = Tags.empty, + createdBy: Subject = Anonymous, + updatedBy: Subject = Anonymous + ): FileState = { + FileState( + id, + project, + storage, + storageType, + attributes, + tags, + rev, + deprecated, + Instant.EPOCH, + createdBy, + Instant.EPOCH, + updatedBy + ) + } + + def resourceFor( + id: Iri, + project: ProjectRef, + storage: ResourceRef.Revision, + attributes: FileAttributes, + storageType: StorageType = StorageType.DiskStorage, + rev: Int = 1, + deprecated: Boolean = false, + tags: Tags = Tags.empty, + createdBy: Subject = Anonymous, + updatedBy: Subject = Anonymous + ): FileResource = + state(id, project, storage, attributes, storageType, rev, deprecated, tags, createdBy, updatedBy).toResource + +} diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesMock.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/BatchFilesMock.scala similarity index 93% rename from delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesMock.scala rename to delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/BatchFilesMock.scala index 991f853bb4..65c3b80c45 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesMock.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/BatchFilesMock.scala @@ -1,10 +1,11 @@ -package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.mocks import cats.data.NonEmptyList import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.FileResource import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch.BatchFiles import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{CopyFileDestination, FileId} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.CopyFileSource import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller import ch.epfl.bluebrain.nexus.delta.sdk.model.IdSegment import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.User diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesRoutesSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesRoutesSpec.scala index 6580477c75..45a1c6add3 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesRoutesSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesRoutesSpec.scala @@ -6,42 +6,35 @@ import akka.http.scaladsl.server.Route import cats.data.NonEmptyList import ch.epfl.bluebrain.nexus.delta.plugins.storage.files import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch.BatchFiles +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.generators.FileGen +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.mocks.BatchFilesMock +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.mocks.BatchFilesMock.BatchFilesCopyFilesCalled import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileId -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.BatchFilesMock.BatchFilesCopyFilesCalled -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.BatchFilesRoutesSpec.BatchFilesRoutesGenerators -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{contexts => fileContexts, schemas, FileFixtures, FileGen, FileResource} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{contexts => fileContexts, FileFixtures, FileResource} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StorageFixtures -import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary -import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.nxv import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.{ContextValue, RemoteContextResolution} import ch.epfl.bluebrain.nexus.delta.sdk.IndexingAction import ch.epfl.bluebrain.nexus.delta.sdk.acls.model.AclAddress import ch.epfl.bluebrain.nexus.delta.sdk.acls.{AclCheck, AclSimpleCheck} import ch.epfl.bluebrain.nexus.delta.sdk.directives.DeltaSchemeDirectives -import ch.epfl.bluebrain.nexus.delta.sdk.generators.ProjectGen import ch.epfl.bluebrain.nexus.delta.sdk.identities.IdentitiesDummy import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller import ch.epfl.bluebrain.nexus.delta.sdk.model.{IdSegment, IdSegmentRef} import ch.epfl.bluebrain.nexus.delta.sdk.permissions.model.Permission import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContextDummy -import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.{ApiMappings, Project, ProjectContext} +import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.Project import ch.epfl.bluebrain.nexus.delta.sdk.utils.BaseRouteSpec import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.User +import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef import ch.epfl.bluebrain.nexus.delta.sourcing.model.Tag.UserTag -import ch.epfl.bluebrain.nexus.delta.sourcing.model.{Label, ProjectRef, ResourceRef} -import ch.epfl.bluebrain.nexus.testkit.Generators import io.circe.Json import io.circe.syntax.KeyOps -import org.scalatest.{Assertion, Assertions} +import org.scalatest.Assertion import scala.collection.mutable.ListBuffer -class BatchFilesRoutesSpec - extends BaseRouteSpec - with StorageFixtures - with FileFixtures - with BatchFilesRoutesGenerators { +class BatchFilesRoutesSpec extends BaseRouteSpec with StorageFixtures with FileFixtures with FileGen { implicit override def rcr: RemoteContextResolution = RemoteContextResolution.fixedIO( @@ -193,30 +186,6 @@ class BatchFilesRoutesSpec } object BatchFilesRoutesSpec { - - trait BatchFilesRoutesGenerators { self: Generators with FileFixtures with Assertions => - def genProjectRef(): ProjectRef = ProjectRef.unsafe(genString(), genString()) - def genProject(): Project = { - val projRef = genProjectRef() - val apiMappings = ApiMappings("file" -> schemas.files) - ProjectGen.project(projRef.project.value, projRef.organization.value, base = nxv.base, mappings = apiMappings) - } - - def genUser(realmLabel: Label): User = User(genString(), realmLabel) - def genFilesIdsInProject(projRef: ProjectRef): NonEmptyList[FileId] = - NonEmptyList.of(genString(), genString()).map(id => FileId(id, projRef)) - def genFileIdWithRev(projRef: ProjectRef): FileId = FileId(genString(), 4, projRef) - def genFileIdWithTag(projRef: ProjectRef): FileId = FileId(genString(), UserTag.unsafe(genString()), projRef) - - def genFileResource(fileId: FileId, context: ProjectContext): FileResource = - FileGen.resourceFor( - fileId.id.value.toIri(context.apiMappings, context.base).getOrElse(fail(s"Bad file $fileId")), - fileId.project, - ResourceRef.Revision(Iri.unsafe(genString()), 1), - attributes(genString()) - ) - } - def mkBulkCopyPayload(sourceProj: ProjectRef, sourceFileIds: NonEmptyList[FileId]): Json = Json.obj("sourceProjectRef" := sourceProj.toString, "files" := mkSourceFilesPayload(sourceFileIds)) From cbe30653d0c98fd18a53f7db2115ce55ac16c247 Mon Sep 17 00:00:00 2001 From: dantb Date: Mon, 11 Dec 2023 17:25:23 +0100 Subject: [PATCH 12/18] Delete previous copy impl, test and fix error mapping in routes --- .../plugins/storage/StoragePluginModule.scala | 23 ++-- .../delta/plugins/storage/files/Files.scala | 87 +------------- .../storage/files/batch/BatchCopy.scala | 20 ++-- .../storage/files/batch/BatchFiles.scala | 5 +- .../storage/files/model/FileRejection.scala | 16 +-- .../storage/files/routes/FilesRoutes.scala | 36 +----- .../storage/storages/model/Storage.scala | 10 +- .../storages/operations/CopyFiles.scala | 27 ----- .../operations/StorageFileRejection.scala | 27 ++++- .../storages/operations/disk/DiskCopy.scala | 55 --------- .../disk/DiskStorageCopyFiles.scala | 81 ++++++------- .../operations/remote/RemoteDiskCopy.scala | 60 ---------- .../remote/RemoteDiskStorageCopyFiles.scala | 80 +++++++------ .../model}/RemoteDiskCopyDetails.scala | 2 +- .../plugins/storage/files/FilesSpec.scala | 111 +----------------- .../storage/files/mocks/BatchFilesMock.scala | 8 +- .../files/routes/BatchFilesRoutesSpec.scala | 76 +++++++++++- .../files/routes/FilesRoutesSpec.scala | 52 +------- 18 files changed, 238 insertions(+), 538 deletions(-) delete mode 100644 delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/CopyFiles.scala delete mode 100644 delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskCopy.scala delete mode 100644 delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskCopy.scala rename delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/{ => client/model}/RemoteDiskCopyDetails.scala (94%) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala index 64f0d24a32..24d93bba58 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala @@ -17,8 +17,8 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StoragesConfig.Sto import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.contexts.{storages => storageCtxId, storagesMetadata => storageMetaCtxId} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageAccess -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.DiskCopy -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.RemoteDiskCopy +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.DiskStorageCopyFiles +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.RemoteDiskStorageCopyFiles import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.RemoteDiskStorageClient import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.routes.StoragesRoutes import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.schemas.{storage => storagesSchemaId} @@ -183,8 +183,7 @@ class StoragePluginModule(priority: Int) extends ModuleDef { storageTypeConfig, cfg.files, remoteDiskStorageClient, - clock, - TransactionalFileCopier.mk() + clock )( uuidF, as @@ -197,18 +196,18 @@ class StoragePluginModule(priority: Int) extends ModuleDef { make[TransactionalFileCopier].fromValue(TransactionalFileCopier.mk()) - make[DiskCopy].from { copier: TransactionalFileCopier => DiskCopy.mk(copier) } + make[DiskStorageCopyFiles].from { copier: TransactionalFileCopier => DiskStorageCopyFiles.mk(copier) } - make[RemoteDiskCopy].from { client: RemoteDiskStorageClient => RemoteDiskCopy.mk(client) } + make[RemoteDiskStorageCopyFiles].from { client: RemoteDiskStorageClient => RemoteDiskStorageCopyFiles.mk(client) } make[BatchCopy].from { ( - files: Files, - storages: Storages, - storagesStatistics: StoragesStatistics, - diskCopy: DiskCopy, - remoteDiskCopy: RemoteDiskCopy, - uuidF: UUIDF + files: Files, + storages: Storages, + storagesStatistics: StoragesStatistics, + diskCopy: DiskStorageCopyFiles, + remoteDiskCopy: RemoteDiskStorageCopyFiles, + uuidF: UUIDF ) => BatchCopy.mk(files, storages, storagesStatistics, diskCopy, remoteDiskCopy)(uuidF) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala index d996607d5b..8ca4291d63 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala @@ -4,12 +4,11 @@ import akka.actor.typed.ActorSystem import akka.actor.{ActorSystem => ClassicActorSystem} import akka.http.scaladsl.model.ContentTypes.`application/octet-stream` import akka.http.scaladsl.model.{BodyPartEntity, ContentType, HttpEntity, Uri} -import cats.data.NonEmptyList import cats.effect.{Clock, IO} import cats.syntax.all._ import ch.epfl.bluebrain.nexus.delta.kernel.cache.LocalCache import ch.epfl.bluebrain.nexus.delta.kernel.kamon.KamonMetricComponent -import ch.epfl.bluebrain.nexus.delta.kernel.utils.{TransactionalFileCopier, UUIDF} +import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF import ch.epfl.bluebrain.nexus.delta.kernel.{Logger, RetryStrategy} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.Files._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest.{ComputedDigest, NotComputedDigest} @@ -18,10 +17,9 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileCommand._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileEvent._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model._ -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.CopyFileSource import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.schemas.{files => fileSchema} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StoragesConfig.{RemoteDiskStorageConfig, StorageTypeConfig} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageRejection.{DifferentStorageType, InvalidStorageType, StorageFetchRejection, StorageIsDeprecated} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageRejection.{StorageFetchRejection, StorageIsDeprecated} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{DigestAlgorithm, Storage, StorageRejection, StorageType} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.{FetchAttributeRejection, FetchFileRejection, SaveFileRejection} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations._ @@ -63,8 +61,7 @@ final class Files( storages: Storages, storagesStatistics: StoragesStatistics, remoteDiskStorageClient: RemoteDiskStorageClient, - config: StorageTypeConfig, - copier: TransactionalFileCopier + config: StorageTypeConfig )(implicit uuidF: UUIDF, system: ClassicActorSystem @@ -198,78 +195,6 @@ final class Files( } yield res }.span("createLink") - def copyFiles( - source: CopyFileSource, - dest: CopyFileDestination - )(implicit c: Caller): IO[NonEmptyList[FileResource]] = { - for { - (pc, destStorageRef, destStorage) <- fetchDestinationStorage(dest) - copyDetails <- source.files.traverse(fetchCopyDetails(destStorage, _)) - _ <- validateSpaceOnStorage(destStorage, copyDetails.map(_.sourceAttributes.bytes)) - destFilesAttributes <- CopyFiles(destStorage, remoteDiskStorageClient, copier).apply(copyDetails) - fileResources <- evalCreateCommands(pc, dest, destStorageRef, destStorage.tpe, destFilesAttributes) - } yield fileResources - } - - private def evalCreateCommands( - pc: ProjectContext, - dest: CopyFileDestination, - destStorageRef: ResourceRef.Revision, - destStorageTpe: StorageType, - destFilesAttributes: NonEmptyList[FileAttributes] - )(implicit c: Caller): IO[NonEmptyList[FileResource]] = - destFilesAttributes.traverse { destFileAttributes => - for { - iri <- generateId(pc) - command = CreateFile(iri, dest.project, destStorageRef, destStorageTpe, destFileAttributes, c.subject, dest.tag) - resource <- eval(command).onError { e => - logger.error(e)( - s"Failed to save event during file copy, saved file must be manually deleted: $command" - ) - } - } yield resource - } - - private def validateSpaceOnStorage(destStorage: Storage, sourcesBytes: NonEmptyList[Long]): IO[Unit] = for { - space <- storagesStatistics.getStorageAvailableSpace(destStorage) - maxSize = destStorage.storageValue.maxFileSize - _ <- IO.raiseWhen(sourcesBytes.exists(_ > maxSize))(FileTooLarge(maxSize, space)) - totalSize = sourcesBytes.toList.sum - _ <- IO.raiseWhen(space.exists(_ < totalSize))(FileTooLarge(maxSize, space)) - } yield () - - private def fetchCopyDetails(destStorage: Storage, fileId: FileId)(implicit c: Caller) = - for { - (file, sourceStorage) <- fetchSourceFile(fileId) - _ <- validateStorageTypeForCopy(file.storageType, destStorage) - destinationDesc <- FileDescription(file.attributes.filename, file.attributes.mediaType) - } yield CopyFileDetails(destinationDesc, file.attributes, sourceStorage) - - private def fetchSourceFile(id: FileId)(implicit c: Caller) = - for { - file <- fetch(id) - sourceStorage <- storages.fetch(file.value.storage, id.project) - _ <- validateAuth(id.project, sourceStorage.value.storageValue.readPermission) - } yield (file.value, sourceStorage.value) - - private def fetchDestinationStorage( - dest: CopyFileDestination - )(implicit c: Caller): IO[(ProjectContext, ResourceRef.Revision, Storage)] = - for { - pc <- fetchContext.onCreate(dest.project) - (destStorageRef, destStorage) <- fetchAndValidateActiveStorage(dest.storage, dest.project, pc) - } yield (pc, destStorageRef, destStorage) - - private def validateStorageTypeForCopy(source: StorageType, destination: Storage): IO[Unit] = - IO.raiseWhen(source == StorageType.S3Storage)( - WrappedStorageRejection( - InvalidStorageType(destination.id, source, Set(StorageType.DiskStorage, StorageType.RemoteDiskStorage)) - ) - ) >> - IO.raiseUnless(source == destination.tpe)( - WrappedStorageRejection(DifferentStorageType(destination.id, found = destination.tpe, expected = source)) - ) - /** * Update an existing file * @@ -842,8 +767,7 @@ object Files { storageTypeConfig: StorageTypeConfig, config: FilesConfig, remoteDiskStorageClient: RemoteDiskStorageClient, - clock: Clock[IO], - copier: TransactionalFileCopier + clock: Clock[IO] )(implicit uuidF: UUIDF, as: ActorSystem[Nothing] @@ -857,8 +781,7 @@ object Files { storages, storagesStatistics, remoteDiskStorageClient, - storageTypeConfig, - copier + storageTypeConfig ) } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopy.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopy.scala index c7cb08b3bc..9dfcfa83de 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopy.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopy.scala @@ -12,8 +12,10 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.{Dis import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageRejection.DifferentStorageType import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{Storage, StorageType} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.CopyFileRejection -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.{DiskCopy, DiskCopyDetails} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.{RemoteDiskCopy, RemoteDiskCopyDetails} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.CopyFileRejection.{TotalCopySizeTooLarge, SourceFileTooLarge} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.{DiskCopyDetails, DiskStorageCopyFiles} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.RemoteDiskStorageCopyFiles +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.model.RemoteDiskCopyDetails import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{Storages, StoragesStatistics} import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller import ch.epfl.bluebrain.nexus.delta.sourcing.model.Label @@ -27,11 +29,11 @@ trait BatchCopy { object BatchCopy { def mk( - files: Files, - storages: Storages, - storagesStatistics: StoragesStatistics, - diskCopy: DiskCopy, - remoteDiskCopy: RemoteDiskCopy + files: Files, + storages: Storages, + storagesStatistics: StoragesStatistics, + diskCopy: DiskStorageCopyFiles, + remoteDiskCopy: RemoteDiskStorageCopyFiles )(implicit uuidF: UUIDF): BatchCopy = new BatchCopy { override def copyFiles(source: CopyFileSource, destStorage: Storage)(implicit @@ -60,9 +62,9 @@ object BatchCopy { private def validateSpaceOnStorage(destStorage: Storage, sourcesBytes: NonEmptyList[Long]): IO[Unit] = for { space <- storagesStatistics.getStorageAvailableSpace(destStorage) maxSize = destStorage.storageValue.maxFileSize - _ <- IO.raiseWhen(sourcesBytes.exists(_ > maxSize))(FileTooLarge(maxSize, space)) + _ <- IO.raiseWhen(sourcesBytes.exists(_ > maxSize))(SourceFileTooLarge(maxSize, destStorage.id)) totalSize = sourcesBytes.toList.sum - _ <- IO.raiseWhen(space.exists(_ < totalSize))(FileTooLarge(maxSize, space)) + _ <- space.collectFirst { case s if s < totalSize => IO.raiseError(TotalCopySizeTooLarge(totalSize, s, destStorage.id)) }.getOrElse(IO.unit) } yield () private def fetchDiskCopyDetails(destStorage: DiskStorage, fileId: FileId)(implicit c: Caller) = diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFiles.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFiles.scala index a8f2628fd0..ae8feb06d1 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFiles.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFiles.scala @@ -2,15 +2,18 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch import cats.data.NonEmptyList import cats.effect.IO +import cats.implicits._ import ch.epfl.bluebrain.nexus.delta.kernel.Logger import ch.epfl.bluebrain.nexus.delta.kernel.kamon.KamonMetricComponent import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.Files.entityType import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileCommand._ +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection.CopyRejection import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.CopyFileSource import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{FetchFileStorage, FileResource} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageType +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.CopyFileRejection import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller import ch.epfl.bluebrain.nexus.delta.sdk.implicits._ @@ -43,7 +46,7 @@ object BatchFiles { for { pc <- fetchContext.onCreate(dest.project) (destStorageRef, destStorage) <- fetchFileStorage.fetchAndValidateActiveStorage(dest.storage, dest.project, pc) - destFilesAttributes <- batchCopy.copyFiles(source, destStorage) + destFilesAttributes <- batchCopy.copyFiles(source, destStorage).adaptError { case e: CopyFileRejection => CopyRejection(source.project, dest.project, destStorage.id, e)} fileResources <- evalCreateCommands(pc, dest, destStorageRef, destStorage.tpe, destFilesAttributes) } yield fileResources }.span("copyFiles") diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileRejection.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileRejection.scala index 87f82ffa92..3ab9cdbb4f 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileRejection.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileRejection.scala @@ -8,7 +8,7 @@ import ch.epfl.bluebrain.nexus.delta.kernel.utils.ClassUtils import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageRejection import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageRejection.StorageFetchRejection import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.{FetchFileRejection, SaveFileRejection} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.{CopyFileRejection, FetchFileRejection, SaveFileRejection} import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.ContextValue @@ -235,16 +235,13 @@ object FileRejection { final case class LinkRejection(id: Iri, storageId: Iri, rejection: StorageFileRejection) extends FileRejection(s"File '$id' could not be linked using storage '$storageId'", Some(rejection.loggedDetails)) - /** - * Rejection returned when interacting with the storage operations bundle to copy a file already in storage - */ final case class CopyRejection( - sourceId: Iri, - sourceStorageId: Iri, + sourceProj: ProjectRef, + destProject: ProjectRef, destStorageId: Iri, - rejection: StorageFileRejection + rejection: CopyFileRejection ) extends FileRejection( - s"File '$sourceId' could not be copied from storage '$sourceStorageId' to storage '$destStorageId'", + s"Failed to copy files from $sourceProj to storage $destStorageId in project $destProject", Some(rejection.loggedDetails) ) @@ -272,6 +269,8 @@ object FileRejection { obj.add(keywords.tpe, ClassUtils.simpleName(rejection).asJson).add("details", rejection.loggedDetails.asJson) case LinkRejection(_, _, rejection) => obj.add(keywords.tpe, ClassUtils.simpleName(rejection).asJson).add("details", rejection.loggedDetails.asJson) + case CopyRejection(_, _, _, rejection) => + obj.add(keywords.tpe, ClassUtils.simpleName(rejection).asJson).add("details", rejection.loggedDetails.asJson) case ProjectContextRejection(rejection) => rejection.asJsonObject case IncorrectRev(provided, expected) => obj.add("provided", provided.asJson).add("expected", expected.asJson) case _: FileNotFound => obj.add(keywords.tpe, "ResourceNotFound".asJson) @@ -296,6 +295,7 @@ object FileRejection { // If this happens it signifies a system problem rather than the user having made a mistake case FetchRejection(_, _, FetchFileRejection.FileNotFound(_)) => (StatusCodes.InternalServerError, Seq.empty) case SaveRejection(_, _, SaveFileRejection.ResourceAlreadyExists(_)) => (StatusCodes.Conflict, Seq.empty) + case CopyRejection(_, _, _, rejection) => (rejection.status, Seq.empty) case FetchRejection(_, _, _) => (StatusCodes.InternalServerError, Seq.empty) case SaveRejection(_, _, _) => (StatusCodes.InternalServerError, Seq.empty) case _ => (StatusCodes.BadRequest, Seq.empty) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutes.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutes.scala index a408c83007..358fcab7b9 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutes.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutes.scala @@ -5,18 +5,16 @@ import akka.http.scaladsl.model.Uri.Path import akka.http.scaladsl.model.headers.Accept import akka.http.scaladsl.model.{ContentType, MediaRange} import akka.http.scaladsl.server._ -import cats.data.{EitherT, NonEmptyList} import cats.effect.IO import cats.syntax.all._ -import ch.epfl.bluebrain.nexus.delta.kernel.Logger + import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection._ -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{CopyFileDestination, File, FileId, FileRejection} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{File, FileId, FileRejection} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.permissions.{read => Read, write => Write} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.FilesRoutes._ -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{contexts, schemas, FileResource, Files} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{schemas, FileResource, Files} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StoragesConfig.StorageTypeConfig -import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.{ContextValue, RemoteContextResolution} -import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.encoder.JsonLdEncoder +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.RemoteContextResolution import ch.epfl.bluebrain.nexus.delta.rdf.utils.JsonKeyOrdering import ch.epfl.bluebrain.nexus.delta.sdk._ import ch.epfl.bluebrain.nexus.delta.sdk.acls.AclCheck @@ -28,7 +26,7 @@ import ch.epfl.bluebrain.nexus.delta.sdk.fusion.FusionConfig import ch.epfl.bluebrain.nexus.delta.sdk.identities.Identities import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller import ch.epfl.bluebrain.nexus.delta.sdk.implicits._ -import ch.epfl.bluebrain.nexus.delta.sdk.jsonld.BulkOperationResults + import ch.epfl.bluebrain.nexus.delta.sdk.model.routes.Tag import ch.epfl.bluebrain.nexus.delta.sdk.model.{BaseUri, IdSegment} import ch.epfl.bluebrain.nexus.delta.sourcing.model.Tag.UserTag @@ -68,14 +66,9 @@ final class FilesRoutes( ) extends AuthDirectives(identities, aclCheck) with CirceUnmarshalling { self => - private val logger = Logger[FilesRoutes] - import baseUri.prefixSegment import schemeDirectives._ - implicit val nelEnc: JsonLdEncoder[NonEmptyList[FileResource]] = - JsonLdEncoder.computeFromCirce[NonEmptyList[FileResource]](Files.context) - def routes: Route = (baseUriPrefix(baseUri.prefix) & replaceUri("files", schemas.files)) { pathPrefix("files") { @@ -101,13 +94,6 @@ final class FilesRoutes( .attemptNarrow[FileRejection] ) }, - // Bulk create files by copying from another project - entity(as[CopyFileSource]) { c: CopyFileSource => - val copyTo = CopyFileDestination(projectRef, storage, tag) - implicit val bulkOpJsonLdEnc: JsonLdEncoder[BulkOperationResults[FileResource]] = - BulkOperationResults.searchResultsJsonLdEncoder(ContextValue(contexts.files)) - emit(Created, copyFile(mode, c, copyTo)) - }, // Create a file without id segment extractRequestEntity { entity => emit( @@ -253,18 +239,6 @@ final class FilesRoutes( } } - private def copyFile(mode: IndexingMode, c: CopyFileSource, copyTo: CopyFileDestination)(implicit - caller: Caller - ): IO[Either[FileRejection, BulkOperationResults[FileResource]]] = - (for { - _ <- EitherT.right(aclCheck.authorizeForOr(c.project, Read)(AuthorizationFailed(c.project.project, Read))) - result <- EitherT(files.copyFiles(c, copyTo).attemptNarrow[FileRejection]) - bulkResults = BulkOperationResults(result.toList) - _ <- EitherT.right[FileRejection](logger.info(s"Indexing and returning bulk results $bulkResults")) - _ <- EitherT.right[FileRejection](result.traverse(index(copyTo.project, _, mode))) - _ <- EitherT.right[FileRejection](logger.info(s"Finished indexing")) - } yield bulkResults).value - def fetch(id: FileId)(implicit caller: Caller): Route = (headerValueByType(Accept) & varyAcceptHeaders) { case accept if accept.mediaRanges.exists(metadataMediaRanges.contains) => diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/Storage.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/Storage.scala index 110d08db5f..9c89ac50ef 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/Storage.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/Storage.scala @@ -1,16 +1,15 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model import akka.actor.ActorSystem -import ch.epfl.bluebrain.nexus.delta.kernel.utils.TransactionalFileCopier import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StoragesConfig.StorageTypeConfig import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.Metadata import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageValue.{DiskStorageValue, RemoteDiskStorageValue, S3StorageValue} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations._ -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.{DiskStorageCopyFiles, DiskStorageFetchFile, DiskStorageSaveFile} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.{DiskStorageFetchFile, DiskStorageSaveFile} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.RemoteDiskStorageClient import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.s3.{S3StorageFetchFile, S3StorageLinkFile, S3StorageSaveFile} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{contexts, Storages} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{Storages, contexts} import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.ContextValue import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.JsonLdContext.keywords @@ -89,8 +88,6 @@ object Storage { def saveFile(implicit as: ActorSystem): SaveFile = new DiskStorageSaveFile(this) - - def copyFiles(copier: TransactionalFileCopier): CopyFiles = new DiskStorageCopyFiles(this, copier) } /** @@ -140,9 +137,6 @@ object Storage { def linkFile(client: RemoteDiskStorageClient): LinkFile = new RemoteDiskStorageLinkFile(this, client) - def copyFiles(client: RemoteDiskStorageClient): CopyFiles = - new RemoteDiskStorageCopyFiles(this, client) - def fetchComputedAttributes(client: RemoteDiskStorageClient): FetchAttributes = new RemoteStorageFetchAttributes(value, client) } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/CopyFiles.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/CopyFiles.scala deleted file mode 100644 index 3e2382b2a6..0000000000 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/CopyFiles.scala +++ /dev/null @@ -1,27 +0,0 @@ -package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations - -import cats.data.NonEmptyList -import cats.effect.IO -import ch.epfl.bluebrain.nexus.delta.kernel.utils.TransactionalFileCopier -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{CopyFileDetails, FileAttributes} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{Storage, StorageType} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.CopyFileRejection -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.RemoteDiskStorageClient - -trait CopyFiles { - def apply(copyDetails: NonEmptyList[CopyFileDetails]): IO[NonEmptyList[FileAttributes]] -} - -object CopyFiles { - - def apply(storage: Storage, client: RemoteDiskStorageClient, copier: TransactionalFileCopier): CopyFiles = - storage match { - case storage: Storage.DiskStorage => storage.copyFiles(copier) - case storage: Storage.S3Storage => unsupported(storage.tpe) - case storage: Storage.RemoteDiskStorage => storage.copyFiles(client) - } - - private def unsupported(storageType: StorageType): CopyFiles = - _ => IO.raiseError(CopyFileRejection.UnsupportedOperation(storageType)) - -} diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/StorageFileRejection.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/StorageFileRejection.scala index a3511184cc..f6ec729105 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/StorageFileRejection.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/StorageFileRejection.scala @@ -1,7 +1,10 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageType +import akka.http.scaladsl.model.StatusCodes import ch.epfl.bluebrain.nexus.delta.kernel.error.Rejection +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageType +import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri +import ch.epfl.bluebrain.nexus.delta.sdk.marshalling.HttpResponseFields /** * Enumeration of Storage rejections related to file operations. @@ -71,14 +74,26 @@ object StorageFileRejection { sealed abstract class CopyFileRejection(loggedDetails: String) extends StorageFileRejection(loggedDetails) object CopyFileRejection { - - /** - * Rejection performing this operation because the storage does not support it - */ final case class UnsupportedOperation(tpe: StorageType) - extends FetchAttributeRejection( + extends CopyFileRejection( s"Copying a file attributes is not supported for storages of type '${tpe.iri}'" ) + + final case class SourceFileTooLarge(maxSize: Long, storageId: Iri) + extends CopyFileRejection( + s"Source file size exceeds maximum $maxSize on destination storage $storageId" + ) + + final case class TotalCopySizeTooLarge(totalSize: Long, spaceLeft: Long, storageId: Iri) + extends CopyFileRejection( + s"Combined size of source files ($totalSize) exceeds space ($spaceLeft) on destination storage $storageId" + ) + + implicit val statusCodes: HttpResponseFields[CopyFileRejection] = HttpResponseFields { + case _: UnsupportedOperation => StatusCodes.BadRequest + case _: SourceFileTooLarge => StatusCodes.BadRequest + case _: TotalCopySizeTooLarge => StatusCodes.BadRequest + } } /** diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskCopy.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskCopy.scala deleted file mode 100644 index 67dff2f52c..0000000000 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskCopy.scala +++ /dev/null @@ -1,55 +0,0 @@ -package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk - -import akka.http.scaladsl.model.Uri -import cats.data.NonEmptyList -import cats.effect.IO -import ch.epfl.bluebrain.nexus.delta.kernel.utils.{CopyBetween, TransactionalFileCopier} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.DiskStorage -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.DiskStorageSaveFile.computeLocation -import fs2.io.file.Path - -import java.nio.file - -trait DiskCopy { - def copyFiles(destStorage: DiskStorage, details: NonEmptyList[DiskCopyDetails]): IO[NonEmptyList[FileAttributes]] -} - -object DiskCopy { - def mk(copier: TransactionalFileCopier): DiskCopy = new DiskCopy { - - def copyFiles(destStorage: DiskStorage, details: NonEmptyList[DiskCopyDetails]): IO[NonEmptyList[FileAttributes]] = - details - .traverse(mkCopyDetailsAndDestAttributes(destStorage, _)) - .flatMap { copyDetailsAndDestAttributes => - val copyDetails = copyDetailsAndDestAttributes.map(_._1) - val destAttrs = copyDetailsAndDestAttributes.map(_._2) - copier.copyAll(copyDetails).as(destAttrs) - } - - private def mkCopyDetailsAndDestAttributes(destStorage: DiskStorage, copyFile: DiskCopyDetails) = - for { - sourcePath <- absoluteDiskPathFromAttributes(copyFile.sourceAttributes) - (destPath, destRelativePath) <- computeDestLocation(destStorage, copyFile) - destAttr = mkDestAttributes(copyFile, destPath, destRelativePath) - copyDetails <- absoluteDiskPathFromAttributes(destAttr).map { dest => - CopyBetween(Path.fromNioPath(sourcePath), Path.fromNioPath(dest)) - } - } yield (copyDetails, destAttr) - - private def computeDestLocation(destStorage: DiskStorage, cd: DiskCopyDetails): IO[(file.Path, file.Path)] = - computeLocation(destStorage.project, destStorage.value, cd.destinationDesc.uuid, cd.destinationDesc.filename) - - private def mkDestAttributes(cd: DiskCopyDetails, destPath: file.Path, destRelativePath: file.Path) = - FileAttributes( - uuid = cd.destinationDesc.uuid, - location = Uri(destPath.toUri.toString), - path = Uri.Path(destRelativePath.toString), - filename = cd.destinationDesc.filename, - mediaType = cd.sourceAttributes.mediaType, - bytes = cd.sourceAttributes.bytes, - digest = cd.sourceAttributes.digest, - origin = cd.sourceAttributes.origin - ) - } -} diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageCopyFiles.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageCopyFiles.scala index 4397e18d32..75f95d6a9a 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageCopyFiles.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageCopyFiles.scala @@ -4,51 +4,52 @@ import akka.http.scaladsl.model.Uri import cats.data.NonEmptyList import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.kernel.utils.{CopyBetween, TransactionalFileCopier} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{CopyFileDetails, FileAttributes} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.DiskStorage -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.CopyFiles import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.DiskStorageSaveFile.computeLocation import fs2.io.file.Path import java.nio.file -class DiskStorageCopyFiles(storage: DiskStorage, copier: TransactionalFileCopier) extends CopyFiles { - - override def apply(details: NonEmptyList[CopyFileDetails]): IO[NonEmptyList[FileAttributes]] = - details - .traverse(mkCopyDetailsAndDestAttributes) - .flatMap { copyDetailsAndDestAttributes => - val copyDetails = copyDetailsAndDestAttributes.map(_._1) - val destAttrs = copyDetailsAndDestAttributes.map(_._2) - copier.copyAll(copyDetails).as(destAttrs) - } - - private def mkCopyDetailsAndDestAttributes(copyFile: CopyFileDetails) = - for { - sourcePath <- absoluteDiskPathFromAttributes(copyFile.sourceAttributes) - (destPath, destRelativePath) <- computeLocation( - storage.project, - storage.value, - copyFile.destinationDesc.uuid, - copyFile.destinationDesc.filename - ) - destAttr = mkDestAttributes(copyFile, destPath, destRelativePath) - copyDetails <- absoluteDiskPathFromAttributes(destAttr).map { dest => - CopyBetween(Path.fromNioPath(sourcePath), Path.fromNioPath(dest)) - } - } yield (copyDetails, destAttr) - - private def mkDestAttributes(copyFile: CopyFileDetails, destPath: file.Path, destRelativePath: file.Path) = { - val dest = copyFile.destinationDesc - FileAttributes( - uuid = dest.uuid, - location = Uri(destPath.toUri.toString), - path = Uri.Path(destRelativePath.toString), - filename = dest.filename, - mediaType = copyFile.sourceAttributes.mediaType, - bytes = copyFile.sourceAttributes.bytes, - digest = copyFile.sourceAttributes.digest, - origin = copyFile.sourceAttributes.origin - ) +trait DiskStorageCopyFiles { + def copyFiles(destStorage: DiskStorage, details: NonEmptyList[DiskCopyDetails]): IO[NonEmptyList[FileAttributes]] +} + +object DiskStorageCopyFiles { + def mk(copier: TransactionalFileCopier): DiskStorageCopyFiles = new DiskStorageCopyFiles { + + def copyFiles(destStorage: DiskStorage, details: NonEmptyList[DiskCopyDetails]): IO[NonEmptyList[FileAttributes]] = + details + .traverse(mkCopyDetailsAndDestAttributes(destStorage, _)) + .flatMap { copyDetailsAndDestAttributes => + val copyDetails = copyDetailsAndDestAttributes.map(_._1) + val destAttrs = copyDetailsAndDestAttributes.map(_._2) + copier.copyAll(copyDetails).as(destAttrs) + } + + private def mkCopyDetailsAndDestAttributes(destStorage: DiskStorage, copyFile: DiskCopyDetails) = + for { + sourcePath <- absoluteDiskPathFromAttributes(copyFile.sourceAttributes) + (destPath, destRelativePath) <- computeDestLocation(destStorage, copyFile) + destAttr = mkDestAttributes(copyFile, destPath, destRelativePath) + copyDetails <- absoluteDiskPathFromAttributes(destAttr).map { dest => + CopyBetween(Path.fromNioPath(sourcePath), Path.fromNioPath(dest)) + } + } yield (copyDetails, destAttr) + + private def computeDestLocation(destStorage: DiskStorage, cd: DiskCopyDetails): IO[(file.Path, file.Path)] = + computeLocation(destStorage.project, destStorage.value, cd.destinationDesc.uuid, cd.destinationDesc.filename) + + private def mkDestAttributes(cd: DiskCopyDetails, destPath: file.Path, destRelativePath: file.Path) = + FileAttributes( + uuid = cd.destinationDesc.uuid, + location = Uri(destPath.toUri.toString), + path = Uri.Path(destRelativePath.toString), + filename = cd.destinationDesc.filename, + mediaType = cd.sourceAttributes.mediaType, + bytes = cd.sourceAttributes.bytes, + digest = cd.sourceAttributes.digest, + origin = cd.sourceAttributes.origin + ) } } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskCopy.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskCopy.scala deleted file mode 100644 index f887b98322..0000000000 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskCopy.scala +++ /dev/null @@ -1,60 +0,0 @@ -package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote - -import akka.http.scaladsl.model.Uri -import akka.http.scaladsl.model.Uri.Path -import cats.data.NonEmptyList -import cats.effect.IO -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.RemoteDiskStorage -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.SaveFile.intermediateFolders -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.RemoteDiskStorageClient -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.model.RemoteDiskCopyPaths - -trait RemoteDiskCopy { - def copyFiles( - destStorage: RemoteDiskStorage, - copyDetails: NonEmptyList[RemoteDiskCopyDetails] - ): IO[NonEmptyList[FileAttributes]] -} - -object RemoteDiskCopy { - - def mk(client: RemoteDiskStorageClient): RemoteDiskCopy = new RemoteDiskCopy { - def copyFiles( - destStorage: RemoteDiskStorage, - copyDetails: NonEmptyList[RemoteDiskCopyDetails] - ): IO[NonEmptyList[FileAttributes]] = { - - val paths = remoteDiskCopyPaths(destStorage, copyDetails) - - client.copyFiles(destStorage.value.folder, paths)(destStorage.value.endpoint).map { destPaths => - copyDetails.zip(paths).zip(destPaths).map { case ((copyDetails, remoteCopyPaths), absoluteDestPath) => - mkDestAttributes(copyDetails, remoteCopyPaths.destPath, absoluteDestPath) - } - } - } - } - - private def mkDestAttributes(cd: RemoteDiskCopyDetails, relativeDestPath: Path, absoluteDestPath: Uri) = { - val destDesc = cd.destinationDesc - val sourceAttr = cd.sourceAttributes - FileAttributes( - uuid = destDesc.uuid, - location = absoluteDestPath, - path = relativeDestPath, - filename = destDesc.filename, - mediaType = destDesc.mediaType, - bytes = sourceAttr.bytes, - digest = sourceAttr.digest, - origin = sourceAttr.origin - ) - } - - private def remoteDiskCopyPaths(destStorage: RemoteDiskStorage, copyDetails: NonEmptyList[RemoteDiskCopyDetails]) = - copyDetails.map { cd => - val destDesc = cd.destinationDesc - val destinationPath = Uri.Path(intermediateFolders(destStorage.project, destDesc.uuid, destDesc.filename)) - val sourcePath = cd.sourceAttributes.path - RemoteDiskCopyPaths(cd.sourceBucket, sourcePath, destinationPath) - } -} diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageCopyFiles.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageCopyFiles.scala index 442b715caf..3d84070755 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageCopyFiles.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageCopyFiles.scala @@ -1,56 +1,60 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote import akka.http.scaladsl.model.Uri +import akka.http.scaladsl.model.Uri.Path import cats.data.NonEmptyList import cats.effect.IO -import ch.epfl.bluebrain.nexus.delta.kernel.Logger -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{CopyFileDetails, FileAttributes} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.RemoteDiskStorage -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageValue -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.CopyFiles import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.SaveFile.intermediateFolders import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.RemoteDiskStorageClient -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.model.RemoteDiskCopyPaths +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.model.{RemoteDiskCopyDetails, RemoteDiskCopyPaths} -class RemoteDiskStorageCopyFiles( - destStorage: RemoteDiskStorage, - client: RemoteDiskStorageClient -) extends CopyFiles { +trait RemoteDiskStorageCopyFiles { + def copyFiles( + destStorage: RemoteDiskStorage, + copyDetails: NonEmptyList[RemoteDiskCopyDetails] + ): IO[NonEmptyList[FileAttributes]] +} - private val logger = Logger[RemoteDiskStorageCopyFiles] +object RemoteDiskStorageCopyFiles { - override def apply(copyDetails: NonEmptyList[CopyFileDetails]): IO[NonEmptyList[FileAttributes]] = { - val maybePaths = copyDetails.traverse { cd => - val destinationPath = - Uri.Path(intermediateFolders(destStorage.project, cd.destinationDesc.uuid, cd.destinationDesc.filename)) - val sourcePath = cd.sourceAttributes.path + def mk(client: RemoteDiskStorageClient): RemoteDiskStorageCopyFiles = new RemoteDiskStorageCopyFiles { + def copyFiles( + destStorage: RemoteDiskStorage, + copyDetails: NonEmptyList[RemoteDiskCopyDetails] + ): IO[NonEmptyList[FileAttributes]] = { - val thingy = cd.sourceStorage.storageValue match { - case remote: StorageValue.RemoteDiskStorageValue => IO(remote.folder) - case other => IO.raiseError(new Exception(s"Invalid storage type for remote copy: $other")) - } - thingy.map(sourceBucket => RemoteDiskCopyPaths(sourceBucket, sourcePath, destinationPath)) - } + val paths = remoteDiskCopyPaths(destStorage, copyDetails) - maybePaths.flatMap { paths => - logger.info(s"DTBDTB REMOTE doing copy with ${destStorage.value.folder} and $paths") >> - client.copyFiles(destStorage.value.folder, paths)(destStorage.value.endpoint).flatMap { destPaths => - logger.info(s"DTBDTB REMOTE received destPaths ${destPaths}").as { - copyDetails.zip(paths).zip(destPaths).map { case ((cd, x), destinationPath) => - FileAttributes( - uuid = cd.destinationDesc.uuid, - location = destinationPath, - path = x.destPath, - filename = cd.destinationDesc.filename, - mediaType = cd.destinationDesc.mediaType, - bytes = cd.sourceAttributes.bytes, - digest = cd.sourceAttributes.digest, - origin = cd.sourceAttributes.origin - ) - } - } + client.copyFiles(destStorage.value.folder, paths)(destStorage.value.endpoint).map { destPaths => + copyDetails.zip(paths).zip(destPaths).map { case ((copyDetails, remoteCopyPaths), absoluteDestPath) => + mkDestAttributes(copyDetails, remoteCopyPaths.destPath, absoluteDestPath) } + } } } + private def mkDestAttributes(cd: RemoteDiskCopyDetails, relativeDestPath: Path, absoluteDestPath: Uri) = { + val destDesc = cd.destinationDesc + val sourceAttr = cd.sourceAttributes + FileAttributes( + uuid = destDesc.uuid, + location = absoluteDestPath, + path = relativeDestPath, + filename = destDesc.filename, + mediaType = destDesc.mediaType, + bytes = sourceAttr.bytes, + digest = sourceAttr.digest, + origin = sourceAttr.origin + ) + } + + private def remoteDiskCopyPaths(destStorage: RemoteDiskStorage, copyDetails: NonEmptyList[RemoteDiskCopyDetails]) = + copyDetails.map { cd => + val destDesc = cd.destinationDesc + val destinationPath = Uri.Path(intermediateFolders(destStorage.project, destDesc.uuid, destDesc.filename)) + val sourcePath = cd.sourceAttributes.path + RemoteDiskCopyPaths(cd.sourceBucket, sourcePath, destinationPath) + } } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskCopyDetails.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/model/RemoteDiskCopyDetails.scala similarity index 94% rename from delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskCopyDetails.scala rename to delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/model/RemoteDiskCopyDetails.scala index 710a621d51..1ad264e161 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskCopyDetails.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/model/RemoteDiskCopyDetails.scala @@ -1,4 +1,4 @@ -package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote +package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.model import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes, FileDescription} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.RemoteDiskStorage diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala index bc5d9d223b..4693747853 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala @@ -1,23 +1,19 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.files import akka.actor.typed.scaladsl.adapter._ -import akka.actor.{typed, ActorSystem} +import akka.actor.{ActorSystem, typed} import akka.http.scaladsl.model.ContentTypes.`text/plain(UTF-8)` import akka.http.scaladsl.model.Uri import akka.testkit.TestKit -import cats.data.NonEmptyList import cats.effect.IO -import cats.effect.unsafe.implicits.global import ch.epfl.bluebrain.nexus.delta.kernel.http.MediaTypeDetectorConfig -import ch.epfl.bluebrain.nexus.delta.kernel.utils.TransactionalFileCopier import ch.epfl.bluebrain.nexus.delta.plugins.storage.RemoteContextResolutionFixture import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.generators.FileGen import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest.NotComputedDigest import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes.FileAttributesOrigin.Storage import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection._ -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{CopyFileDestination, FileAttributes, FileId, FileRejection} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.CopyFileSource -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageRejection.{DifferentStorageType, StorageNotFound} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes, FileId, FileRejection} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageRejection.StorageNotFound import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageType.{RemoteDiskStorage => RemoteStorageType} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{StorageRejection, StorageStatEntry, StorageType} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.AkkaSourceHelpers @@ -49,7 +45,6 @@ import org.scalatest.concurrent.Eventually import org.scalatest.{Assertion, DoNotDiscover} import java.net.URLDecoder -import java.util.UUID @DoNotDiscover class FilesSpec(docker: RemoteStorageDocker) @@ -140,8 +135,7 @@ class FilesSpec(docker: RemoteStorageDocker) cfg, FilesConfig(eventLogConfig, MediaTypeDetectorConfig.Empty), remoteDiskStorageClient, - clock, - TransactionalFileCopier.mk() + clock ) def fileId(file: String): FileId = FileId(file, projectRef) @@ -451,103 +445,6 @@ class FilesSpec(docker: RemoteStorageDocker) } } - "copying a file" should { - - "succeed from disk storage based on a tag" in { - // TODO: adding uuids whenever we want a new independent test is not sustainable. If we truly want to test this every - // time we should generate a new "Files" with a new UUIDF (and other dependencies we want to control). - // Alternatively we could normalise the expected values to not care about any generated Ids - val newFileUuid = UUID.randomUUID() - withUUIDF(newFileUuid) { - val source = CopyFileSource(projectRef, NonEmptyList.of(FileId("file1", tag, projectRef))) - val destination = CopyFileDestination(projectRefOrg2, Some(diskId), None) - - val expectedDestId = project2.base.iri / newFileUuid.toString - val expectedFilename = "myfile.txt" - val expectedAttr = attributes(filename = expectedFilename, projRef = projectRefOrg2, id = newFileUuid) - val expected = mkResource(expectedDestId, projectRefOrg2, diskRev, expectedAttr) - - val actual = files.copyFiles(source, destination).unsafeRunSync() - actual shouldEqual NonEmptyList.of(expected) - - val fetched = files.fetch(FileId(newFileUuid.toString, projectRefOrg2)).accepted - fetched shouldEqual expected - } - } - - "succeed from disk storage based on a rev and should tag the new file" in { - val newFileUuid = UUID.randomUUID() - withUUIDF(newFileUuid) { - val source = CopyFileSource(projectRef, NonEmptyList.of(FileId("file1", 2, projectRef))) - val newTag = UserTag.unsafe(genString()) - val destination = CopyFileDestination(projectRefOrg2, Some(diskId), Some(newTag)) - - val expectedDestId = project2.base.iri / newFileUuid.toString - val expectedFilename = "file.txt" - val expectedAttr = attributes(filename = expectedFilename, projRef = projectRefOrg2, id = newFileUuid) - val expected = mkResource(expectedDestId, projectRefOrg2, diskRev, expectedAttr, tags = Tags(newTag -> 1)) - - val actual = files.copyFiles(source, destination).accepted - actual shouldEqual NonEmptyList.of(expected) - - val fetchedByTag = files.fetch(FileId(newFileUuid.toString, newTag, projectRefOrg2)).accepted - fetchedByTag shouldEqual expected - } - } - - "reject if the source file doesn't exist" in { - val destination = CopyFileDestination(projectRefOrg2, None, None) - val source = CopyFileSource(projectRef, NonEmptyList.of(fileIdIri(nxv + "other"))) - files.copyFiles(source, destination).rejectedWith[FileNotFound] - } - - "reject if the destination storage doesn't exist" in { - val destination = CopyFileDestination(projectRefOrg2, Some(storage), None) - val source = CopyFileSource(projectRef, NonEmptyList.of(fileId("file1"))) - files.copyFiles(source, destination).rejected shouldEqual - WrappedStorageRejection(StorageNotFound(storageIri, projectRefOrg2)) - } - - "reject if copying between different storage types" in { - val expectedError = DifferentStorageType(remoteIdIri, StorageType.RemoteDiskStorage, StorageType.DiskStorage) - val destination = CopyFileDestination(projectRefOrg2, Some(remoteId), None) - val source = CopyFileSource(projectRef, NonEmptyList.of(FileId("file1", projectRef))) - files.copyFiles(source, destination).rejected shouldEqual - WrappedStorageRejection(expectedError) - } - - val smallDiskCapacity = 9 - val smallDiskMaxSize = 5 - - "reject if total size of source files exceed remaining available space on the destination storage" in { - givenAFileWithSize(5) { fileId1 => - givenAFileWithSize(5) { fileId2 => - val smallDiskPayload = - diskFieldsJson deepMerge json"""{"capacity": $smallDiskCapacity, "maxFileSize": $smallDiskMaxSize, "volume": "$path"}""" - storages.create(smallDiskId, projectRefOrg2, smallDiskPayload).accepted - - val source = CopyFileSource(projectRef, NonEmptyList.of(fileId1, fileId2)) - val destination = CopyFileDestination(projectRefOrg2, Some(smallDiskId), None) - val expectedError = FileTooLarge(smallDiskMaxSize.toLong, Some(smallDiskCapacity.toLong)) - - files.copyFiles(source, destination).rejected shouldEqual expectedError - } - } - } - - "reject if any of the files exceed max file size of the destination storage" in { - givenAFileWithSize(1) { fileId1 => - givenAFileWithSize(smallDiskMaxSize + 1) { fileId2 => - val source = CopyFileSource(projectRef, NonEmptyList.of(fileId1, fileId2)) - val destination = CopyFileDestination(projectRefOrg2, Some(smallDiskId), None) - val expectedError = FileTooLarge(smallDiskMaxSize.toLong, Some(smallDiskCapacity.toLong)) - - files.copyFiles(source, destination).rejected shouldEqual expectedError - } - } - } - } - "deleting a tag" should { "succeed" in { val expected = mkResource(file1, projectRef, diskRev, attributes(), rev = 4) diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/BatchFilesMock.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/BatchFilesMock.scala index 65c3b80c45..705741ad5e 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/BatchFilesMock.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/BatchFilesMock.scala @@ -20,12 +20,16 @@ object BatchFilesMock { def withStubbedCopyFiles( stubbed: NonEmptyList[FileResource], - buffer: ListBuffer[BatchFilesCopyFilesCalled] + events: ListBuffer[BatchFilesCopyFilesCalled] ): BatchFiles = withMockedCopyFiles((source, dest) => - c => IO(buffer.addOne(BatchFilesCopyFilesCalled(source, dest, c))).as(stubbed) + c => IO(events.addOne(BatchFilesCopyFilesCalled(source, dest, c))).as(stubbed) ) + def withError(e: Throwable, events: ListBuffer[BatchFilesCopyFilesCalled]): BatchFiles = withMockedCopyFiles((source, dest) => + c => IO(events.addOne(BatchFilesCopyFilesCalled(source, dest, c))) >> IO.raiseError(e) + ) + def withMockedCopyFiles( copyFilesMock: (CopyFileSource, CopyFileDestination) => Caller => IO[NonEmptyList[FileResource]] ): BatchFiles = new BatchFiles { diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesRoutesSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesRoutesSpec.scala index 45a1c6add3..00aef9481b 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesRoutesSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesRoutesSpec.scala @@ -1,17 +1,22 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes -import akka.http.scaladsl.model.StatusCodes import akka.http.scaladsl.model.headers.OAuth2BearerToken +import akka.http.scaladsl.model.{StatusCode, StatusCodes} import akka.http.scaladsl.server.Route import cats.data.NonEmptyList +import ch.epfl.bluebrain.nexus.delta.kernel.utils.ClassUtils import ch.epfl.bluebrain.nexus.delta.plugins.storage.files import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch.BatchFiles import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.generators.FileGen import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.mocks.BatchFilesMock import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.mocks.BatchFilesMock.BatchFilesCopyFilesCalled -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileId +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection.{CopyRejection, FileNotFound, WrappedStorageRejection} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileId, FileRejection} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{contexts => fileContexts, FileFixtures, FileResource} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StorageFixtures +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageRejection.{DifferentStorageType, StorageNotFound} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageType +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.CopyFileRejection.{SourceFileTooLarge, TotalCopySizeTooLarge, UnsupportedOperation} import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.{ContextValue, RemoteContextResolution} import ch.epfl.bluebrain.nexus.delta.sdk.IndexingAction @@ -78,7 +83,7 @@ class BatchFilesRoutesSpec extends BaseRouteSpec with StorageFixtures with FileF testBulkCopySucceedsForStubbedFiles(sourceProj, sourceFileIds, destTag = Some(destTag)) } - "be rejected for a user without read permission on the source project" in { + "return 403 for a user without read permission on the source project" in { val (sourceProj, destProj, user) = (genProject(), genProject(), genUser(realm)) val sourceFileIds = genFilesIdsInProject(sourceProj.ref) @@ -90,7 +95,7 @@ class BatchFilesRoutesSpec extends BaseRouteSpec with StorageFixtures with FileF } } - "be rejected if tag and rev are present simultaneously for a source file" in { + "return 400 if tag and rev are present simultaneously for a source file" in { val (sourceProj, destProj, user) = (genProject(), genProject(), genUser(realm)) val route = mkRoute(BatchFilesMock.unimplemented, sourceProj, user, permissions = Set()) @@ -101,6 +106,69 @@ class BatchFilesRoutesSpec extends BaseRouteSpec with StorageFixtures with FileF response.status shouldBe StatusCodes.BadRequest } } + + "return 400 for copy errors raised by batch file logic" in { + val unsupportedStorageType = UnsupportedOperation(StorageType.S3Storage) + val fileTooLarge = SourceFileTooLarge(12, genIri()) + val totalSizeTooLarge = TotalCopySizeTooLarge(1L, 2L, genIri()) + + val errors = List(unsupportedStorageType, fileTooLarge, totalSizeTooLarge) + .map(CopyRejection(genProjectRef(), genProjectRef(), genIri(), _)) + + forAll(errors) { error => + val (sourceProj, destProj, user) = (genProject(), genProject(), genUser(realm)) + val sourceFileIds = genFilesIdsInProject(sourceProj.ref) + val events = ListBuffer.empty[BatchFilesCopyFilesCalled] + val batchFiles = BatchFilesMock.withError(error, events) + + val route = mkRoute(batchFiles, sourceProj, user, permissions = Set(files.permissions.read)) + val payload = BatchFilesRoutesSpec.mkBulkCopyPayload(sourceProj.ref, sourceFileIds) + + callBulkCopyEndpoint(route, destProj.ref, payload, user) { + response.status shouldBe StatusCodes.BadRequest + response.asJson shouldBe errorJson(error, Some(ClassUtils.simpleName(error.rejection)), error.loggedDetails) + } + } + } + + "map other file rejections to the correct response" in { + val storageNotFound = WrappedStorageRejection(StorageNotFound(genIri(), genProjectRef())) + val differentStorageType = + WrappedStorageRejection(DifferentStorageType(genIri(), StorageType.DiskStorage, StorageType.RemoteDiskStorage)) + val fileNotFound = FileNotFound(genIri(), genProjectRef()) + + val fileRejections: List[(FileRejection, StatusCode, Json)] = List( + (storageNotFound, StatusCodes.NotFound, errorJson(storageNotFound, Some("ResourceNotFound"))), + (fileNotFound, StatusCodes.NotFound, errorJson(fileNotFound, Some("ResourceNotFound"))), + (differentStorageType, StatusCodes.BadRequest, errorJson(differentStorageType, Some("DifferentStorageType"))) + ) + + forAll(fileRejections) { case (error, expectedStatus, expectedJson) => + val (sourceProj, destProj, user) = (genProject(), genProject(), genUser(realm)) + val sourceFileIds = genFilesIdsInProject(sourceProj.ref) + val events = ListBuffer.empty[BatchFilesCopyFilesCalled] + val batchFiles = BatchFilesMock.withError(error, events) + + val route = mkRoute(batchFiles, sourceProj, user, permissions = Set(files.permissions.read)) + val payload = BatchFilesRoutesSpec.mkBulkCopyPayload(sourceProj.ref, sourceFileIds) + + callBulkCopyEndpoint(route, destProj.ref, payload, user) { + response.status shouldBe expectedStatus + response.asJson shouldBe expectedJson + } + } + } + } + + def errorJson(t: Throwable, specificType: Option[String] = None, details: Option[String] = None): Json = { + val detailsObj = details.fold(Json.obj())(d => Json.obj("details" := d)) + detailsObj.deepMerge( + Json.obj( + "@context" := Vocabulary.contexts.error.toString, + "@type" := specificType.getOrElse(ClassUtils.simpleName(t)), + "reason" := t.getMessage + ) + ) } def mkRoute( diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala index 5484cbd2bd..dffb05c9bd 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala @@ -9,13 +9,13 @@ import akka.http.scaladsl.model.{StatusCodes, Uri} import akka.http.scaladsl.server.Route import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.kernel.http.MediaTypeDetectorConfig -import ch.epfl.bluebrain.nexus.delta.kernel.utils.{ClasspathResourceLoader, TransactionalFileCopier} +import ch.epfl.bluebrain.nexus.delta.kernel.utils.ClasspathResourceLoader import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest.ComputedDigest import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes, FileId, FileRejection} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{contexts => fileContexts, permissions, FileFixtures, Files, FilesConfig} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{FileFixtures, Files, FilesConfig, permissions, contexts => fileContexts} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{StorageRejection, StorageStatEntry, StorageType} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.RemoteDiskStorageClient -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{contexts => storageContexts, permissions => storagesPermissions, StorageFixtures, Storages, StoragesConfig, StoragesStatistics} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{StorageFixtures, Storages, StoragesConfig, StoragesStatistics, contexts => storageContexts, permissions => storagesPermissions} import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.rdf.RdfMediaTypes.`application/ld+json` import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary @@ -43,11 +43,9 @@ import ch.epfl.bluebrain.nexus.testkit.ce.IOFromMap import ch.epfl.bluebrain.nexus.testkit.errors.files.FileErrors.{fileAlreadyExistsError, fileIsNotDeprecatedError} import ch.epfl.bluebrain.nexus.testkit.scalatest.ce.CatsIOValues import io.circe.Json -import io.circe.syntax.{EncoderOps, KeyOps} +import io.circe.syntax.EncoderOps import org.scalatest._ -import java.util.UUID - class FilesRoutesSpec extends BaseRouteSpec with CancelAfterFailure @@ -138,8 +136,7 @@ class FilesRoutesSpec config, FilesConfig(eventLogConfig, MediaTypeDetectorConfig.Empty), remoteDiskStorageClient, - clock, - TransactionalFileCopier.mk() + clock )(uuidF, typedSystem) private val groupDirectives = DeltaSchemeDirectives( @@ -366,45 +363,6 @@ class FilesRoutesSpec } } - "copy a file" in { - givenAFileInProject(projectRef.toString) { oldFileId => - val newFileUUId = UUID.randomUUID() - withUUIDF(newFileUUId) { - val newFileId = newFileUUId.toString - val json = - Json.obj("sourceProjectRef" := projectRef, "files" := Json.arr(Json.obj("sourceFileId" := oldFileId))) - - Post(s"/v1/files/${projectRefOrg2.toString}", json.toEntity) ~> asWriter ~> routes ~> check { - status shouldEqual StatusCodes.Created - val expectedId = project2.base.iri / newFileId - val expectedAttr = attributes(filename = oldFileId, id = newFileUUId) - val expectedFile = fileMetadata(projectRefOrg2, expectedId, expectedAttr, diskIdRev) - val expected = bulkOperationResponse(1, List(expectedFile)) - response.asJson shouldEqual expected - } - } - } - } - - "reject file copy request if tag and rev are present simultaneously" in { - givenAFileInProject(projectRef.toString) { oldFileId => - val json = Json.obj( - "sourceProjectRef" := projectRef.toString, - "files" := Json.arr( - Json.obj( - "sourceFileId" := oldFileId, - "sourceTag" := "mytag", - "sourceRev" := 3 - ) - ) - ) - - Post(s"/v1/files/${projectRefOrg2.toString}", json.toEntity) ~> asWriter ~> routes ~> check { - status shouldEqual StatusCodes.UnsupportedMediaType - } - } - } - "deprecate a file" in { givenAFile { id => Delete(s"/v1/files/org/proj/$id?rev=1") ~> asWriter ~> routes ~> check { From 2477b8f4d2a709632d389a928777b8580b967524 Mon Sep 17 00:00:00 2001 From: dantb Date: Mon, 11 Dec 2023 17:54:13 +0100 Subject: [PATCH 13/18] Test rejections are mapped correctly --- .../storage/files/BatchFilesSpec.scala | 41 ++++++++++++++----- .../storage/files/mocks/BatchCopyMock.scala | 34 +++++++++++++++ 2 files changed, 64 insertions(+), 11 deletions(-) create mode 100644 delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/BatchCopyMock.scala diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/BatchFilesSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/BatchFilesSpec.scala index 5a68019d00..49570f2a79 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/BatchFilesSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/BatchFilesSpec.scala @@ -1,16 +1,19 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.files -import cats.data.NonEmptyList import cats.effect.IO +import cats.implicits.catsSyntaxApplicativeError import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.BatchFilesSpec._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch.{BatchCopy, BatchFiles} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.generators.FileGen +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.mocks.BatchCopyMock import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileCommand.CreateFile -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes, FileCommand, FileRejection} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection.CopyRejection +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileCommand, FileRejection} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.CopyFileSource import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StorageFixtures import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.CopyFileRejection.TotalCopySizeTooLarge import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller import ch.epfl.bluebrain.nexus.delta.sdk.model.IdSegment import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.{Project, ProjectContext} @@ -24,13 +27,13 @@ import scala.collection.mutable.ListBuffer class BatchFilesSpec extends NexusSuite with StorageFixtures with Generators with FileFixtures with FileGen { - test("batch copying files should fetch storage, perform copy and evaluate create file commands") { + test("batch copying should fetch storage, perform copy and evaluate create file commands") { val events = ListBuffer.empty[Event] val destProj: Project = genProject() val (destStorageRef, destStorage) = (genRevision(), genStorage(destProj.ref, diskVal)) val fetchFileStorage = mockFetchFileStorage(destStorageRef, destStorage.storage, events) val stubbedDestAttributes = genAttributes() - val batchCopy = mockBatchCopy(events, stubbedDestAttributes) + val batchCopy = BatchCopyMock.withStubbedCopyFiles(events, stubbedDestAttributes) val destFileUUId = UUID.randomUUID() // Not testing UUID generation, same for all of them val batchFiles: BatchFiles = mkBatchFiles(events, destProj, destFileUUId, fetchFileStorage, batchCopy) @@ -53,6 +56,29 @@ class BatchFilesSpec extends NexusSuite with StorageFixtures with Generators wit assertEquals(events.toList, expectedEvents) } + test("copy rejections should be mapped to a file rejection") { + val events = ListBuffer.empty[Event] + val destProj: Project = genProject() + val (destStorageRef, destStorage) = (genRevision(), genStorage(destProj.ref, diskVal)) + val fetchFileStorage = mockFetchFileStorage(destStorageRef, destStorage.storage, events) + val error = TotalCopySizeTooLarge(1L, 2L, genIri()) + val batchCopy = BatchCopyMock.withError(error, events) + + val batchFiles: BatchFiles = mkBatchFiles(events, destProj, UUID.randomUUID(), fetchFileStorage, batchCopy) + implicit val c: Caller = Caller(genUser(), Set()) + val sourceProj = genProject() + val (source, destination) = + (genCopyFileSource(sourceProj.ref), genCopyFileDestination(destProj.ref, destStorage.storage)) + val result = batchFiles.copyFiles(source, destination).attemptNarrow[CopyRejection].accepted + + assertEquals(result, Left(CopyRejection(sourceProj.ref, destProj.ref, destStorage.id, error))) + + val expectedActiveStorageFetched = ActiveStorageFetched(destination.storage, destProj.ref, destProj.context, c) + val expectedBatchCopyCalled = BatchCopyCalled(source, destStorage.storage, c) + val expectedEvents = List(expectedActiveStorageFetched, expectedBatchCopyCalled) + assertEquals(events.toList, expectedEvents) + } + def mockFetchFileStorage( storageRef: ResourceRef.Revision, storage: Storage, @@ -64,13 +90,6 @@ class BatchFilesSpec extends NexusSuite with StorageFixtures with Generators wit IO(events.addOne(ActiveStorageFetched(storageIdOpt, ref, pc, caller))).as(storageRef -> storage) } - def mockBatchCopy(events: ListBuffer[Event], stubbedAttr: NonEmptyList[FileAttributes]) = new BatchCopy { - override def copyFiles(source: CopyFileSource, destStorage: Storage)(implicit - c: Caller - ): IO[NonEmptyList[FileAttributes]] = - IO(events.addOne(BatchCopyCalled(source, destStorage, c))).as(stubbedAttr) - } - def mkBatchFiles( events: ListBuffer[Event], proj: Project, diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/BatchCopyMock.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/BatchCopyMock.scala new file mode 100644 index 0000000000..ff00d49fee --- /dev/null +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/BatchCopyMock.scala @@ -0,0 +1,34 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.mocks + +import cats.data.NonEmptyList +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.BatchFilesSpec.{BatchCopyCalled, Event} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch.BatchCopy +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.CopyFileSource +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.CopyFileRejection +import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller + +import scala.collection.mutable.ListBuffer + +object BatchCopyMock { + + def withError( e: CopyFileRejection, events: ListBuffer[Event]): BatchCopy = + withMockedCopyFiles((source, destStorage) => + caller => IO(events.addOne(BatchCopyCalled(source, destStorage, caller))) >> IO.raiseError(e) + ) + + def withStubbedCopyFiles(events: ListBuffer[Event], stubbedAttr: NonEmptyList[FileAttributes]): BatchCopy = + withMockedCopyFiles((source, destStorage) => + caller => IO(events.addOne(BatchCopyCalled(source, destStorage, caller))).as(stubbedAttr) + ) + + def withMockedCopyFiles(copyFilesMock: (CopyFileSource, Storage) => Caller => IO[NonEmptyList[FileAttributes]]): BatchCopy = + new BatchCopy { + override def copyFiles(source: CopyFileSource, destStorage: Storage)(implicit + c: Caller + ): IO[NonEmptyList[FileAttributes]] = copyFilesMock(source, destStorage)(c) + } + +} From 30f8dac0721aaac04bfe48a29c18afaff5429c38 Mon Sep 17 00:00:00 2001 From: dantb Date: Mon, 11 Dec 2023 18:27:11 +0100 Subject: [PATCH 14/18] Create abstractions for file and storage fetching + narrow dependents to these --- .../plugins/storage/StoragePluginModule.scala | 16 +++---- .../storage/files/FetchFileResource.scala | 14 ++++++ .../delta/plugins/storage/files/Files.scala | 19 +++----- .../storage/files/batch/BatchCopy.scala | 35 +++++++++------ .../storage/files/batch/BatchFiles.scala | 4 +- .../storage/files/model/FileRejection.scala | 4 +- .../storage/storages/FetchStorage.scala | 45 +++++++++++++++++++ .../plugins/storage/storages/Storages.scala | 40 +++-------------- .../storage/storages/model/Storage.scala | 2 +- .../operations/StorageFileRejection.scala | 16 +++---- .../plugins/storage/files/FilesSpec.scala | 2 +- .../storage/files/mocks/BatchCopyMock.scala | 8 ++-- .../storage/files/mocks/BatchFilesMock.scala | 7 +-- .../files/routes/FilesRoutesSpec.scala | 4 +- 14 files changed, 126 insertions(+), 90 deletions(-) create mode 100644 delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FetchFileResource.scala create mode 100644 delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/FetchStorage.scala diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala index 24d93bba58..ba7a17af26 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala @@ -202,15 +202,15 @@ class StoragePluginModule(priority: Int) extends ModuleDef { make[BatchCopy].from { ( - files: Files, - storages: Storages, - storagesStatistics: StoragesStatistics, - diskCopy: DiskStorageCopyFiles, - remoteDiskCopy: RemoteDiskStorageCopyFiles, - uuidF: UUIDF + files: Files, + storages: Storages, + aclCheck: AclCheck, + storagesStatistics: StoragesStatistics, + diskCopy: DiskStorageCopyFiles, + remoteDiskCopy: RemoteDiskStorageCopyFiles, + uuidF: UUIDF ) => - BatchCopy.mk(files, storages, storagesStatistics, diskCopy, remoteDiskCopy)(uuidF) - + BatchCopy.mk(files, storages, aclCheck, storagesStatistics, diskCopy, remoteDiskCopy)(uuidF) } make[BatchFiles].from { diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FetchFileResource.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FetchFileResource.scala new file mode 100644 index 0000000000..50bd71ddd7 --- /dev/null +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FetchFileResource.scala @@ -0,0 +1,14 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files + +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileId + +trait FetchFileResource { + /** + * Fetch the last version of a file + * + * @param id + * the identifier that will be expanded to the Iri of the file with its optional rev/tag + */ + def fetch(id: FileId): IO[FileResource] +} diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala index 8ca4291d63..d0e725f17b 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala @@ -24,7 +24,7 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{DigestAlgor import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.{FetchAttributeRejection, FetchFileRejection, SaveFileRejection} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.RemoteDiskStorageClient -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{Storages, StoragesStatistics} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{FetchStorage, Storages, StoragesStatistics} import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.ContextValue import ch.epfl.bluebrain.nexus.delta.sdk.AkkaSource @@ -58,14 +58,15 @@ final class Files( log: FilesLog, aclCheck: AclCheck, fetchContext: FetchContext[FileRejection], - storages: Storages, + storages: FetchStorage, storagesStatistics: StoragesStatistics, remoteDiskStorageClient: RemoteDiskStorageClient, config: StorageTypeConfig )(implicit uuidF: UUIDF, system: ClassicActorSystem -) extends FetchFileStorage { +) extends FetchFileStorage + with FetchFileResource { implicit private val kamonComponent: KamonMetricComponent = KamonMetricComponent(entityType.value) @@ -378,15 +379,7 @@ final class Files( FetchRejection(fileId, storage.id, e) } - /** - * Fetch the last version of a file - * - * @param id - * the identifier that will be expanded to the Iri of the file with its optional rev/tag - * @param project - * the project where the storage belongs - */ - def fetch(id: FileId): IO[FileResource] = + override def fetch(id: FileId): IO[FileResource] = (for { (iri, _) <- id.expandIri(fetchContext.onRead) state <- fetchState(id, iri) @@ -761,7 +754,7 @@ object Files { def apply( fetchContext: FetchContext[FileRejection], aclCheck: AclCheck, - storages: Storages, + storages: FetchStorage, storagesStatistics: StoragesStatistics, xas: Transactors, storageTypeConfig: StorageTypeConfig, diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopy.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopy.scala index 9dfcfa83de..8a8a0a5492 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopy.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopy.scala @@ -4,7 +4,7 @@ import cats.data.NonEmptyList import cats.effect.IO import cats.implicits.toFunctorOps import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.Files +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.FetchFileResource import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.CopyFileSource @@ -12,11 +12,13 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.{Dis import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageRejection.DifferentStorageType import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{Storage, StorageType} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.CopyFileRejection -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.CopyFileRejection.{TotalCopySizeTooLarge, SourceFileTooLarge} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.CopyFileRejection.{SourceFileTooLarge, TotalCopySizeTooLarge} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.{DiskCopyDetails, DiskStorageCopyFiles} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.RemoteDiskStorageCopyFiles import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.model.RemoteDiskCopyDetails -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{Storages, StoragesStatistics} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{FetchStorage, StoragesStatistics} +import ch.epfl.bluebrain.nexus.delta.sdk.acls.AclCheck +import ch.epfl.bluebrain.nexus.delta.sdk.error.ServiceError.AuthorizationFailed import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller import ch.epfl.bluebrain.nexus.delta.sourcing.model.Label import shapeless.syntax.typeable.typeableOps @@ -29,11 +31,12 @@ trait BatchCopy { object BatchCopy { def mk( - files: Files, - storages: Storages, - storagesStatistics: StoragesStatistics, - diskCopy: DiskStorageCopyFiles, - remoteDiskCopy: RemoteDiskStorageCopyFiles + fetchFile: FetchFileResource, + fetchStorage: FetchStorage, + aclCheck: AclCheck, + storagesStatistics: StoragesStatistics, + diskCopy: DiskStorageCopyFiles, + remoteDiskCopy: RemoteDiskStorageCopyFiles )(implicit uuidF: UUIDF): BatchCopy = new BatchCopy { override def copyFiles(source: CopyFileSource, destStorage: Storage)(implicit @@ -64,7 +67,11 @@ object BatchCopy { maxSize = destStorage.storageValue.maxFileSize _ <- IO.raiseWhen(sourcesBytes.exists(_ > maxSize))(SourceFileTooLarge(maxSize, destStorage.id)) totalSize = sourcesBytes.toList.sum - _ <- space.collectFirst { case s if s < totalSize => IO.raiseError(TotalCopySizeTooLarge(totalSize, s, destStorage.id)) }.getOrElse(IO.unit) + _ <- space + .collectFirst { + case s if s < totalSize => IO.raiseError(TotalCopySizeTooLarge(totalSize, s, destStorage.id)) + } + .getOrElse(IO.unit) } yield () private def fetchDiskCopyDetails(destStorage: DiskStorage, fileId: FileId)(implicit c: Caller) = @@ -98,12 +105,14 @@ object BatchCopy { private def unsupported(tpe: StorageType) = IO.raiseError(CopyFileRejection.UnsupportedOperation(tpe)) - private def fetchFileAndValidateStorage(id: FileId)(implicit c: Caller) = + private def fetchFileAndValidateStorage(id: FileId)(implicit c: Caller) = { for { - file <- files.fetch(id) - sourceStorage <- storages.fetch(file.value.storage, id.project) - _ <- files.validateAuth(id.project, sourceStorage.value.storageValue.readPermission) + file <- fetchFile.fetch(id) + sourceStorage <- fetchStorage.fetch(file.value.storage, id.project) + perm = sourceStorage.value.storageValue.readPermission + _ <- aclCheck.authorizeForOr(id.project, perm)(AuthorizationFailed(id.project, perm)) } yield (file.value, sourceStorage.value) + } } } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFiles.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFiles.scala index ae8feb06d1..99d1d03453 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFiles.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFiles.scala @@ -46,7 +46,9 @@ object BatchFiles { for { pc <- fetchContext.onCreate(dest.project) (destStorageRef, destStorage) <- fetchFileStorage.fetchAndValidateActiveStorage(dest.storage, dest.project, pc) - destFilesAttributes <- batchCopy.copyFiles(source, destStorage).adaptError { case e: CopyFileRejection => CopyRejection(source.project, dest.project, destStorage.id, e)} + destFilesAttributes <- batchCopy.copyFiles(source, destStorage).adaptError { case e: CopyFileRejection => + CopyRejection(source.project, dest.project, destStorage.id, e) + } fileResources <- evalCreateCommands(pc, dest, destStorageRef, destStorage.tpe, destFilesAttributes) } yield fileResources }.span("copyFiles") diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileRejection.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileRejection.scala index 3ab9cdbb4f..4fb4909c6d 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileRejection.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileRejection.scala @@ -269,7 +269,7 @@ object FileRejection { obj.add(keywords.tpe, ClassUtils.simpleName(rejection).asJson).add("details", rejection.loggedDetails.asJson) case LinkRejection(_, _, rejection) => obj.add(keywords.tpe, ClassUtils.simpleName(rejection).asJson).add("details", rejection.loggedDetails.asJson) - case CopyRejection(_, _, _, rejection) => + case CopyRejection(_, _, _, rejection) => obj.add(keywords.tpe, ClassUtils.simpleName(rejection).asJson).add("details", rejection.loggedDetails.asJson) case ProjectContextRejection(rejection) => rejection.asJsonObject case IncorrectRev(provided, expected) => obj.add("provided", provided.asJson).add("expected", expected.asJson) @@ -295,7 +295,7 @@ object FileRejection { // If this happens it signifies a system problem rather than the user having made a mistake case FetchRejection(_, _, FetchFileRejection.FileNotFound(_)) => (StatusCodes.InternalServerError, Seq.empty) case SaveRejection(_, _, SaveFileRejection.ResourceAlreadyExists(_)) => (StatusCodes.Conflict, Seq.empty) - case CopyRejection(_, _, _, rejection) => (rejection.status, Seq.empty) + case CopyRejection(_, _, _, rejection) => (rejection.status, Seq.empty) case FetchRejection(_, _, _) => (StatusCodes.InternalServerError, Seq.empty) case SaveRejection(_, _, _) => (StatusCodes.InternalServerError, Seq.empty) case _ => (StatusCodes.BadRequest, Seq.empty) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/FetchStorage.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/FetchStorage.scala new file mode 100644 index 0000000000..d66b9009c6 --- /dev/null +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/FetchStorage.scala @@ -0,0 +1,45 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages + +import cats.effect.IO +import cats.implicits.catsSyntaxMonadError +import ch.epfl.bluebrain.nexus.delta.kernel.Mapper +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageRejection.StorageFetchRejection +import ch.epfl.bluebrain.nexus.delta.sdk.model.IdSegmentRef +import ch.epfl.bluebrain.nexus.delta.sourcing.model.{ProjectRef, ResourceRef} + +trait FetchStorage { + + /** + * Fetch the storage using the ''resourceRef'' + * + * @param resourceRef + * the storage reference (Latest, Revision or Tag) + * @param project + * the project where the storage belongs + */ + final def fetch[R <: Throwable]( + resourceRef: ResourceRef, + project: ProjectRef + )(implicit rejectionMapper: Mapper[StorageFetchRejection, R]): IO[StorageResource] = + fetch(IdSegmentRef(resourceRef), project).adaptError { case err: StorageFetchRejection => + rejectionMapper.to(err) + } + + /** + * Fetch the last version of a storage + * + * @param id + * the identifier that will be expanded to the Iri of the storage with its optional rev/tag + * @param project + * the project where the storage belongs + */ + def fetch(id: IdSegmentRef, project: ProjectRef): IO[StorageResource] + + /** + * Fetches the default storage for a project. + * + * @param project + * the project where to look for the default storage + */ + def fetchDefault(project: ProjectRef): IO[StorageResource] +} diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/Storages.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/Storages.scala index a21301e8e5..6c40b69935 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/Storages.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/Storages.scala @@ -2,9 +2,9 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages import cats.effect.{Clock, IO} import cats.syntax.all._ +import ch.epfl.bluebrain.nexus.delta.kernel.Logger import ch.epfl.bluebrain.nexus.delta.kernel.kamon.KamonMetricComponent import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF -import ch.epfl.bluebrain.nexus.delta.kernel.{Logger, Mapper} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.Storages._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StoragesConfig.StorageTypeConfig import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageCommand._ @@ -30,7 +30,7 @@ import ch.epfl.bluebrain.nexus.delta.sourcing.ScopedEntityDefinition.Tagger import ch.epfl.bluebrain.nexus.delta.sourcing._ import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Subject import ch.epfl.bluebrain.nexus.delta.sourcing.model.Tag.UserTag -import ch.epfl.bluebrain.nexus.delta.sourcing.model.{EntityType, ProjectRef, ResourceRef} +import ch.epfl.bluebrain.nexus.delta.sourcing.model.{EntityType, ProjectRef} import ch.epfl.bluebrain.nexus.delta.sourcing.stream.Elem import fs2.Stream import io.circe.Json @@ -46,7 +46,7 @@ final class Storages private ( fetchContext: FetchContext[StorageFetchRejection], sourceDecoder: JsonLdSourceResolvingDecoder[StorageRejection, StorageFields], serviceAccount: ServiceAccount -) { +) extends FetchStorage { implicit private val kamonComponent: KamonMetricComponent = KamonMetricComponent(entityType.value) @@ -233,31 +233,7 @@ final class Storages private ( } yield res }.span("deprecateStorage") - /** - * Fetch the storage using the ''resourceRef'' - * - * @param resourceRef - * the storage reference (Latest, Revision or Tag) - * @param project - * the project where the storage belongs - */ - def fetch[R <: Throwable]( - resourceRef: ResourceRef, - project: ProjectRef - )(implicit rejectionMapper: Mapper[StorageFetchRejection, R]): IO[StorageResource] = - fetch(IdSegmentRef(resourceRef), project).adaptError { case err: StorageFetchRejection => - rejectionMapper.to(err) - } - - /** - * Fetch the last version of a storage - * - * @param id - * the identifier that will be expanded to the Iri of the storage with its optional rev/tag - * @param project - * the project where the storage belongs - */ - def fetch(id: IdSegmentRef, project: ProjectRef): IO[StorageResource] = { + override def fetch(id: IdSegmentRef, project: ProjectRef): IO[StorageResource] = { for { pc <- fetchContext.onRead(project) iri <- expandIri(id.value, pc) @@ -277,13 +253,7 @@ final class Storages private ( .currentStates(Scope.Project(project), _.toResource) .filter(_.value.default) - /** - * Fetches the default storage for a project. - * - * @param project - * the project where to look for the default storage - */ - def fetchDefault(project: ProjectRef): IO[StorageResource] = { + override def fetchDefault(project: ProjectRef): IO[StorageResource] = { for { defaultOpt <- fetchDefaults(project).reduce(updatedByDesc.min(_, _)).head.compile.last default <- IO.fromOption(defaultOpt)(DefaultStorageNotFound(project)) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/Storage.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/Storage.scala index 9c89ac50ef..b31392f729 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/Storage.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/Storage.scala @@ -9,7 +9,7 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.{D import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.RemoteDiskStorageClient import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.s3.{S3StorageFetchFile, S3StorageLinkFile, S3StorageSaveFile} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{Storages, contexts} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{contexts, Storages} import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.ContextValue import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.JsonLdContext.keywords diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/StorageFileRejection.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/StorageFileRejection.scala index f6ec729105..69dd3023d3 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/StorageFileRejection.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/StorageFileRejection.scala @@ -80,18 +80,18 @@ object StorageFileRejection { ) final case class SourceFileTooLarge(maxSize: Long, storageId: Iri) - extends CopyFileRejection( - s"Source file size exceeds maximum $maxSize on destination storage $storageId" - ) + extends CopyFileRejection( + s"Source file size exceeds maximum $maxSize on destination storage $storageId" + ) final case class TotalCopySizeTooLarge(totalSize: Long, spaceLeft: Long, storageId: Iri) - extends CopyFileRejection( - s"Combined size of source files ($totalSize) exceeds space ($spaceLeft) on destination storage $storageId" - ) + extends CopyFileRejection( + s"Combined size of source files ($totalSize) exceeds space ($spaceLeft) on destination storage $storageId" + ) implicit val statusCodes: HttpResponseFields[CopyFileRejection] = HttpResponseFields { - case _: UnsupportedOperation => StatusCodes.BadRequest - case _: SourceFileTooLarge => StatusCodes.BadRequest + case _: UnsupportedOperation => StatusCodes.BadRequest + case _: SourceFileTooLarge => StatusCodes.BadRequest case _: TotalCopySizeTooLarge => StatusCodes.BadRequest } } diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala index 4693747853..e5757b214c 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala @@ -1,7 +1,7 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.files import akka.actor.typed.scaladsl.adapter._ -import akka.actor.{ActorSystem, typed} +import akka.actor.{typed, ActorSystem} import akka.http.scaladsl.model.ContentTypes.`text/plain(UTF-8)` import akka.http.scaladsl.model.Uri import akka.testkit.TestKit diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/BatchCopyMock.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/BatchCopyMock.scala index ff00d49fee..93702a52f7 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/BatchCopyMock.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/BatchCopyMock.scala @@ -14,7 +14,7 @@ import scala.collection.mutable.ListBuffer object BatchCopyMock { - def withError( e: CopyFileRejection, events: ListBuffer[Event]): BatchCopy = + def withError(e: CopyFileRejection, events: ListBuffer[Event]): BatchCopy = withMockedCopyFiles((source, destStorage) => caller => IO(events.addOne(BatchCopyCalled(source, destStorage, caller))) >> IO.raiseError(e) ) @@ -24,10 +24,12 @@ object BatchCopyMock { caller => IO(events.addOne(BatchCopyCalled(source, destStorage, caller))).as(stubbedAttr) ) - def withMockedCopyFiles(copyFilesMock: (CopyFileSource, Storage) => Caller => IO[NonEmptyList[FileAttributes]]): BatchCopy = + def withMockedCopyFiles( + copyFilesMock: (CopyFileSource, Storage) => Caller => IO[NonEmptyList[FileAttributes]] + ): BatchCopy = new BatchCopy { override def copyFiles(source: CopyFileSource, destStorage: Storage)(implicit - c: Caller + c: Caller ): IO[NonEmptyList[FileAttributes]] = copyFilesMock(source, destStorage)(c) } diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/BatchFilesMock.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/BatchFilesMock.scala index 705741ad5e..f487252f3e 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/BatchFilesMock.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/BatchFilesMock.scala @@ -26,9 +26,10 @@ object BatchFilesMock { c => IO(events.addOne(BatchFilesCopyFilesCalled(source, dest, c))).as(stubbed) ) - def withError(e: Throwable, events: ListBuffer[BatchFilesCopyFilesCalled]): BatchFiles = withMockedCopyFiles((source, dest) => - c => IO(events.addOne(BatchFilesCopyFilesCalled(source, dest, c))) >> IO.raiseError(e) - ) + def withError(e: Throwable, events: ListBuffer[BatchFilesCopyFilesCalled]): BatchFiles = + withMockedCopyFiles((source, dest) => + c => IO(events.addOne(BatchFilesCopyFilesCalled(source, dest, c))) >> IO.raiseError(e) + ) def withMockedCopyFiles( copyFilesMock: (CopyFileSource, CopyFileDestination) => Caller => IO[NonEmptyList[FileResource]] diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala index dffb05c9bd..901aefa286 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala @@ -12,10 +12,10 @@ import ch.epfl.bluebrain.nexus.delta.kernel.http.MediaTypeDetectorConfig import ch.epfl.bluebrain.nexus.delta.kernel.utils.ClasspathResourceLoader import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest.ComputedDigest import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes, FileId, FileRejection} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{FileFixtures, Files, FilesConfig, permissions, contexts => fileContexts} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{contexts => fileContexts, permissions, FileFixtures, Files, FilesConfig} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{StorageRejection, StorageStatEntry, StorageType} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.RemoteDiskStorageClient -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{StorageFixtures, Storages, StoragesConfig, StoragesStatistics, contexts => storageContexts, permissions => storagesPermissions} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{contexts => storageContexts, permissions => storagesPermissions, StorageFixtures, Storages, StoragesConfig, StoragesStatistics} import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.rdf.RdfMediaTypes.`application/ld+json` import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary From a6f4fe644290b0ca7d7323252fbe811df3de4bbb Mon Sep 17 00:00:00 2001 From: dantb Date: Wed, 13 Dec 2023 14:28:58 +0100 Subject: [PATCH 15/18] Test batch copying logic --- .../storage/files/FetchFileResource.scala | 11 +- .../storage/files/batch/BatchCopy.scala | 44 +-- .../storage/files/batch/BatchCopySuite.scala | 294 ++++++++++++++++++ .../BatchFilesSuite.scala} | 10 +- .../storage/files/generators/FileGen.scala | 32 +- .../storage/files/mocks/BatchCopyMock.scala | 2 +- .../storage/files/mocks/DiskCopyMock.scala | 17 + .../files/mocks/FetchFileResourceMock.scala | 13 + .../files/mocks/FetchStorageMock.scala | 17 + .../storage/files/mocks/RemoteCopyMock.scala | 18 ++ .../files/mocks/StoragesStatisticsMock.scala | 15 + 11 files changed, 439 insertions(+), 34 deletions(-) create mode 100644 delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopySuite.scala rename delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/{BatchFilesSpec.scala => batch/BatchFilesSuite.scala} (95%) create mode 100644 delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/DiskCopyMock.scala create mode 100644 delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/FetchFileResourceMock.scala create mode 100644 delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/FetchStorageMock.scala create mode 100644 delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/RemoteCopyMock.scala create mode 100644 delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/StoragesStatisticsMock.scala diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FetchFileResource.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FetchFileResource.scala index 50bd71ddd7..d77edd4c01 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FetchFileResource.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FetchFileResource.scala @@ -4,11 +4,12 @@ import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileId trait FetchFileResource { + /** - * Fetch the last version of a file - * - * @param id - * the identifier that will be expanded to the Iri of the file with its optional rev/tag - */ + * Fetch the last version of a file + * + * @param id + * the identifier that will be expanded to the Iri of the file with its optional rev/tag + */ def fetch(id: FileId): IO[FileResource] } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopy.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopy.scala index 8a8a0a5492..31ff209144 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopy.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopy.scala @@ -17,10 +17,10 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.{D import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.RemoteDiskStorageCopyFiles import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.model.RemoteDiskCopyDetails import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{FetchStorage, StoragesStatistics} +import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.sdk.acls.AclCheck import ch.epfl.bluebrain.nexus.delta.sdk.error.ServiceError.AuthorizationFailed import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller -import ch.epfl.bluebrain.nexus.delta.sourcing.model.Label import shapeless.syntax.typeable.typeableOps trait BatchCopy { @@ -51,28 +51,33 @@ object BatchCopy { private def copyToRemoteStorage(source: CopyFileSource, dest: RemoteDiskStorage)(implicit c: Caller) = for { remoteCopyDetails <- source.files.traverse(fetchRemoteCopyDetails(dest, _)) - _ <- validateSpaceOnStorage(dest, remoteCopyDetails.map(_.sourceAttributes.bytes)) + _ <- validateFilesForStorage(dest, remoteCopyDetails.map(_.sourceAttributes.bytes)) attributes <- remoteDiskCopy.copyFiles(dest, remoteCopyDetails) } yield attributes private def copyToDiskStorage(source: CopyFileSource, dest: DiskStorage)(implicit c: Caller) = for { diskCopyDetails <- source.files.traverse(fetchDiskCopyDetails(dest, _)) - _ <- validateSpaceOnStorage(dest, diskCopyDetails.map(_.sourceAttributes.bytes)) + _ <- validateFilesForStorage(dest, diskCopyDetails.map(_.sourceAttributes.bytes)) attributes <- diskCopy.copyFiles(dest, diskCopyDetails) } yield attributes - private def validateSpaceOnStorage(destStorage: Storage, sourcesBytes: NonEmptyList[Long]): IO[Unit] = for { - space <- storagesStatistics.getStorageAvailableSpace(destStorage) - maxSize = destStorage.storageValue.maxFileSize - _ <- IO.raiseWhen(sourcesBytes.exists(_ > maxSize))(SourceFileTooLarge(maxSize, destStorage.id)) - totalSize = sourcesBytes.toList.sum - _ <- space - .collectFirst { - case s if s < totalSize => IO.raiseError(TotalCopySizeTooLarge(totalSize, s, destStorage.id)) - } - .getOrElse(IO.unit) - } yield () + private def validateFilesForStorage(destStorage: Storage, sourcesBytes: NonEmptyList[Long]): IO[Unit] = { + val maxSize = destStorage.storageValue.maxFileSize + for { + _ <- IO.raiseWhen(sourcesBytes.exists(_ > maxSize))(SourceFileTooLarge(maxSize, destStorage.id)) + _ <- validateRemainingStorageCapacity(destStorage, sourcesBytes) + } yield () + } + + private def validateRemainingStorageCapacity(destStorage: Storage, sourcesBytes: NonEmptyList[Long]) = + for { + spaceLeft <- storagesStatistics.getStorageAvailableSpace(destStorage) + totalSize = sourcesBytes.toList.sum + _ <- spaceLeft + .collectFirst { case s if s < totalSize => notEnoughSpace(totalSize, s, destStorage.id) } + .getOrElse(IO.unit) + } yield () private def fetchDiskCopyDetails(destStorage: DiskStorage, fileId: FileId)(implicit c: Caller) = for { @@ -85,7 +90,7 @@ object BatchCopy { sourceStorage .narrowTo[DiskStorage] .as(IO.unit) - .getOrElse(IO.raiseError(differentStorageTypeError(destStorage, sourceStorage))) + .getOrElse(differentStorageTypeError(destStorage, sourceStorage)) private def fetchRemoteCopyDetails(destStorage: RemoteDiskStorage, fileId: FileId)(implicit c: Caller) = for { @@ -98,13 +103,16 @@ object BatchCopy { sourceStorage .narrowTo[RemoteDiskStorage] .map(remote => IO.pure(remote.value.folder)) - .getOrElse(IO.raiseError[Label](differentStorageTypeError(destStorage, sourceStorage))) + .getOrElse(differentStorageTypeError(destStorage, sourceStorage)) - private def differentStorageTypeError(destStorage: Storage, sourceStorage: Storage) = - DifferentStorageType(destStorage.id, found = sourceStorage.tpe, expected = destStorage.tpe) + private def differentStorageTypeError[A](destStorage: Storage, sourceStorage: Storage) = + IO.raiseError[A](DifferentStorageType(sourceStorage.id, found = sourceStorage.tpe, expected = destStorage.tpe)) private def unsupported(tpe: StorageType) = IO.raiseError(CopyFileRejection.UnsupportedOperation(tpe)) + private def notEnoughSpace(totalSize: Long, spaceLeft: Long, destStorage: Iri) = + IO.raiseError(TotalCopySizeTooLarge(totalSize, spaceLeft, destStorage)) + private def fetchFileAndValidateStorage(id: FileId)(implicit c: Caller) = { for { file <- fetchFile.fetch(id) diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopySuite.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopySuite.scala new file mode 100644 index 0000000000..525c273cd8 --- /dev/null +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopySuite.scala @@ -0,0 +1,294 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch + +import cats.data.NonEmptyList +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch.BatchCopySuite._ +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.generators.FileGen +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.mocks._ +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes, FileDescription, FileId} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.CopyFileSource +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{FetchFileResource, FileFixtures, FileResource} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages._ +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.{DiskStorage, RemoteDiskStorage, S3Storage} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageRejection.DifferentStorageType +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{Storage, StorageStatEntry, StorageType} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.CopyFileRejection.{SourceFileTooLarge, TotalCopySizeTooLarge, UnsupportedOperation} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.{DiskCopyDetails, DiskStorageCopyFiles} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.RemoteDiskStorageCopyFiles +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.model.RemoteDiskCopyDetails +import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.nxv +import ch.epfl.bluebrain.nexus.delta.sdk.acls.model.AclAddress +import ch.epfl.bluebrain.nexus.delta.sdk.acls.{AclCheck, AclSimpleCheck} +import ch.epfl.bluebrain.nexus.delta.sdk.error.ServiceError.AuthorizationFailed +import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller +import ch.epfl.bluebrain.nexus.delta.sdk.model.{IdSegment, IdSegmentRef, Tags} +import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.User +import ch.epfl.bluebrain.nexus.delta.sourcing.model.{ProjectRef, ResourceRef} +import ch.epfl.bluebrain.nexus.testkit.Generators +import ch.epfl.bluebrain.nexus.testkit.mu.NexusSuite + +import java.util.UUID +import scala.collection.mutable.ListBuffer + +class BatchCopySuite extends NexusSuite with StorageFixtures with Generators with FileFixtures with FileGen { + + private val sourceFileDescUuid = UUID.randomUUID() + implicit private val fixedUUidF: UUIDF = UUIDF.fixed(sourceFileDescUuid) + + private val sourceProj = genProject() + private val sourceFileId = genFileId(sourceProj.ref) + private val source = CopyFileSource(sourceProj.ref, NonEmptyList.of(sourceFileId)) + private val storageStatEntry = StorageStatEntry(files = 10L, spaceUsed = 5L) + private val stubbedFileAttr = NonEmptyList.of(attributes(genString())) + + test("successfully perform disk copy") { + val events = ListBuffer.empty[Event] + val (sourceFileRes, sourceStorage) = genFileResourceAndStorage(sourceFileId, sourceProj.context, diskVal) + val (user, aclCheck) = userAuthorizedOnProjectStorage(sourceStorage.value) + + val batchCopy = mkBatchCopy( + fetchFile = stubbedFetchFile(sourceFileRes, events), + fetchStorage = stubbedFetchStorage(sourceStorage, events), + aclCheck = aclCheck, + stats = stubbedStorageStats(storageStatEntry, events), + diskCopy = stubbedDiskCopy(stubbedFileAttr, events) + ) + val destStorage: DiskStorage = genDiskStorage() + + batchCopy.copyFiles(source, destStorage)(caller(user)).map { obtained => + val obtainedEvents = events.toList + assertEquals(obtained, stubbedFileAttr) + sourceFileWasFetched(obtainedEvents, sourceFileId) + sourceStorageWasFetched(obtainedEvents, sourceFileRes.value.storage, sourceProj.ref) + destinationDiskStorageStatsWereFetched(obtainedEvents, destStorage) + diskCopyWasPerformed(obtainedEvents, destStorage, sourceFileRes.value.attributes) + } + } + + test("successfully perform remote disk copy") { + val events = ListBuffer.empty[Event] + val (sourceFileRes, sourceStorage) = genFileResourceAndStorage(sourceFileId, sourceProj.context, remoteVal) + val (user, aclCheck) = userAuthorizedOnProjectStorage(sourceStorage.value) + + val batchCopy = mkBatchCopy( + fetchFile = stubbedFetchFile(sourceFileRes, events), + fetchStorage = stubbedFetchStorage(sourceStorage, events), + aclCheck = aclCheck, + stats = stubbedStorageStats(storageStatEntry, events), + remoteCopy = stubbedRemoteCopy(stubbedFileAttr, events) + ) + val destStorage: RemoteDiskStorage = genRemoteStorage() + + batchCopy.copyFiles(source, destStorage)(caller(user)).map { obtained => + val obtainedEvents = events.toList + assertEquals(obtained, stubbedFileAttr) + sourceFileWasFetched(obtainedEvents, sourceFileId) + sourceStorageWasFetched(obtainedEvents, sourceFileRes.value.storage, sourceProj.ref) + destinationRemoteStorageStatsWereNotFetched(obtainedEvents) + remoteDiskCopyWasPerformed(obtainedEvents, destStorage, sourceFileRes.value.attributes) + } + } + + test("fail if destination storage is S3") { + val batchCopy = mkBatchCopy() + val (user, destStorage) = (genUser(), genS3Storage()) + val expectedError = UnsupportedOperation(StorageType.S3Storage) + batchCopy.copyFiles(source, destStorage)(caller(user)).interceptEquals(expectedError) + } + + test("fail if a source storage is different to destination storage") { + val events = ListBuffer.empty[Event] + val (sourceFileRes, sourceStorage) = genFileResourceAndStorage(sourceFileId, sourceProj.context, diskVal) + val (user, aclCheck) = userAuthorizedOnProjectStorage(sourceStorage.value) + + val batchCopy = mkBatchCopy( + fetchFile = stubbedFetchFile(sourceFileRes, events), + fetchStorage = stubbedFetchStorage(sourceStorage, events), + aclCheck = aclCheck + ) + val expectedError = DifferentStorageType(sourceStorage.id, StorageType.DiskStorage, StorageType.RemoteDiskStorage) + + batchCopy.copyFiles(source, genRemoteStorage())(caller(user)).interceptEquals(expectedError).map { _ => + val obtainedEvents = events.toList + sourceFileWasFetched(obtainedEvents, sourceFileId) + sourceStorageWasFetched(obtainedEvents, sourceFileRes.value.storage, sourceProj.ref) + } + } + + test("fail if user does not have read access on a source file's storage") { + val events = ListBuffer.empty[Event] + val (sourceFileRes, sourceStorage) = genFileResourceAndStorage(sourceFileId, sourceProj.context, diskVal) + val user = genUser() + val aclCheck = AclSimpleCheck((user, AclAddress.fromProject(sourceProj.ref), Set())).accepted + + val batchCopy = mkBatchCopy( + fetchFile = stubbedFetchFile(sourceFileRes, events), + fetchStorage = stubbedFetchStorage(sourceStorage, events), + aclCheck = aclCheck + ) + + batchCopy.copyFiles(source, genDiskStorage())(caller(user)).intercept[AuthorizationFailed].map { _ => + val obtainedEvents = events.toList + sourceFileWasFetched(obtainedEvents, sourceFileId) + sourceStorageWasFetched(obtainedEvents, sourceFileRes.value.storage, sourceProj.ref) + } + } + + test("fail if a single source file exceeds max size for destination storage") { + val events = ListBuffer.empty[Event] + val (sourceFileRes, sourceStorage) = genFileResourceAndStorage(sourceFileId, sourceProj.context, diskVal, 1000L) + val (user, aclCheck) = userAuthorizedOnProjectStorage(sourceStorage.value) + + val batchCopy = mkBatchCopy( + fetchFile = stubbedFetchFile(sourceFileRes, events), + fetchStorage = stubbedFetchStorage(sourceStorage, events), + aclCheck = aclCheck + ) + val destStorage = genDiskStorage() + val error = SourceFileTooLarge(destStorage.value.maxFileSize, destStorage.id) + + batchCopy.copyFiles(source, destStorage)(caller(user)).interceptEquals(error).map { _ => + val obtainedEvents = events.toList + sourceFileWasFetched(obtainedEvents, sourceFileId) + sourceStorageWasFetched(obtainedEvents, sourceFileRes.value.storage, sourceProj.ref) + } + } + + test("fail if total size of source files is too large for destination disk storage") { + val events = ListBuffer.empty[Event] + val fileSize = 6L + val capacity = 10L + val statEntry = StorageStatEntry(files = 10L, spaceUsed = 1L) + val spaceLeft = capacity - statEntry.spaceUsed + val (sourceFileRes, sourceStorage) = genFileResourceAndStorage(sourceFileId, sourceProj.context, diskVal, fileSize) + val (user, aclCheck) = userAuthorizedOnProjectStorage(sourceStorage.value) + + val batchCopy = mkBatchCopy( + fetchFile = stubbedFetchFile(sourceFileRes, events), + fetchStorage = stubbedFetchStorage(sourceStorage, events), + aclCheck = aclCheck, + stats = stubbedStorageStats(statEntry, events) + ) + + val destStorage = genDiskStorageWithCapacity(capacity) + val error = TotalCopySizeTooLarge(fileSize * 2, spaceLeft, destStorage.id) + val twoFileSource = CopyFileSource(sourceProj.ref, NonEmptyList.of(sourceFileId, sourceFileId)) + + batchCopy.copyFiles(twoFileSource, destStorage)(caller(user)).interceptEquals(error).map { _ => + val obtainedEvents = events.toList + sourceFileWasFetched(obtainedEvents, sourceFileId) + sourceStorageWasFetched(obtainedEvents, sourceFileRes.value.storage, sourceProj.ref) + destinationDiskStorageStatsWereFetched(obtainedEvents, destStorage) + } + } + + private def mkBatchCopy( + fetchFile: FetchFileResource = FetchFileResourceMock.unimplemented, + fetchStorage: FetchStorage = FetchStorageMock.unimplemented, + aclCheck: AclCheck = AclSimpleCheck().accepted, + stats: StoragesStatistics = StoragesStatisticsMock.unimplemented, + diskCopy: DiskStorageCopyFiles = DiskCopyMock.unimplemented, + remoteCopy: RemoteDiskStorageCopyFiles = RemoteCopyMock.unimplemented + ): BatchCopy = BatchCopy.mk(fetchFile, fetchStorage, aclCheck, stats, diskCopy, remoteCopy) + + private def userAuthorizedOnProjectStorage(storage: Storage): (User, AclCheck) = { + val user = genUser() + val permissions = Set(storage.storageValue.readPermission) + (user, AclSimpleCheck((user, AclAddress.fromProject(storage.project), permissions)).accepted) + } + + private def sourceFileWasFetched(events: List[Event], id: FileId) = { + val obtained = events.collectFirst { case f: FetchFileCalled => f } + assertEquals(obtained, Some(FetchFileCalled(id))) + } + + private def sourceStorageWasFetched(events: List[Event], storageRef: ResourceRef.Revision, proj: ProjectRef) = { + val obtained = events.collectFirst { case f: FetchStorageCalled => f } + assertEquals(obtained, Some(FetchStorageCalled(IdSegmentRef(storageRef), proj))) + } + + private def destinationDiskStorageStatsWereFetched(events: List[Event], storage: DiskStorage) = { + val obtained = events.collectFirst { case f: StoragesStatsCalled => f } + assertEquals(obtained, Some(StoragesStatsCalled(storage.id, storage.project))) + } + + private def destinationRemoteStorageStatsWereNotFetched(events: List[Event]) = { + val obtained = events.collectFirst { case f: StoragesStatsCalled => f } + assertEquals(obtained, None) + } + + private def diskCopyWasPerformed(events: List[Event], storage: DiskStorage, sourceAttr: FileAttributes) = { + val expectedDesc = FileDescription(sourceFileDescUuid, sourceAttr.filename, sourceAttr.mediaType) + val expectedDiskCopyDetails = DiskCopyDetails(storage, expectedDesc, sourceAttr) + val obtained = events.collectFirst { case f: DiskCopyCalled => f } + assertEquals(obtained, Some(DiskCopyCalled(storage, NonEmptyList.of(expectedDiskCopyDetails)))) + } + + private def remoteDiskCopyWasPerformed( + events: List[Event], + storage: RemoteDiskStorage, + sourceAttr: FileAttributes + ) = { + val expectedDesc = FileDescription(sourceFileDescUuid, sourceAttr.filename, sourceAttr.mediaType) + val expectedCopyDetails = RemoteDiskCopyDetails(storage, expectedDesc, storage.value.folder, sourceAttr) + val obtained = events.collectFirst { case f: RemoteCopyCalled => f } + assertEquals(obtained, Some(RemoteCopyCalled(storage, NonEmptyList.of(expectedCopyDetails)))) + } + + private def caller(user: User): Caller = Caller(user, Set(user)) + + private def genDiskStorageWithCapacity(capacity: Long) = { + val limitedDiskVal = diskVal.copy(capacity = Some(capacity)) + DiskStorage(nxv + genString(), genProject().ref, limitedDiskVal, Tags.empty, json"""{"disk": "value"}""") + } + + private def genDiskStorage() = genDiskStorageWithCapacity(1000L) + + private def genRemoteStorage() = + RemoteDiskStorage(nxv + genString(), genProject().ref, remoteVal, Tags.empty, json"""{"disk": "value"}""") + + private def genS3Storage() = + S3Storage(nxv + genString(), genProject().ref, s3Val, Tags.empty, json"""{"disk": "value"}""") +} + +object BatchCopySuite { + sealed trait Event + final case class FetchFileCalled(id: FileId) extends Event + final case class FetchStorageCalled(id: IdSegmentRef, project: ProjectRef) extends Event + final case class StoragesStatsCalled(idSegment: IdSegment, project: ProjectRef) extends Event + final case class DiskCopyCalled(destStorage: Storage.DiskStorage, details: NonEmptyList[DiskCopyDetails]) + extends Event + final case class RemoteCopyCalled( + destStorage: Storage.RemoteDiskStorage, + copyDetails: NonEmptyList[RemoteDiskCopyDetails] + ) extends Event + + private def stubbedStorageStats(storageStatEntry: StorageStatEntry, events: ListBuffer[Event]) = + StoragesStatisticsMock.withMockedGet((id, proj) => + addEventAndReturn(events, StoragesStatsCalled(id, proj), storageStatEntry) + ) + + private def stubbedFetchFile(sourceFileRes: FileResource, events: ListBuffer[Event]) = + FetchFileResourceMock.withMockedFetch(id => addEventAndReturn(events, FetchFileCalled(id), sourceFileRes)) + + private def stubbedFetchStorage(storage: StorageResource, events: ListBuffer[Event]) = + FetchStorageMock.withMockedFetch((id, proj) => addEventAndReturn(events, FetchStorageCalled(id, proj), storage)) + + private def stubbedRemoteCopy(stubbedRemoteFileAttr: NonEmptyList[FileAttributes], events: ListBuffer[Event]) = + RemoteCopyMock.withMockedCopy((storage, details) => + addEventAndReturn(events, RemoteCopyCalled(storage, details), stubbedRemoteFileAttr) + ) + + private def stubbedDiskCopy(stubbedDiskFileAttr: NonEmptyList[FileAttributes], events: ListBuffer[Event]) = + DiskCopyMock.withMockedCopy((storage, details) => + addEventAndReturn(events, DiskCopyCalled(storage, details), stubbedDiskFileAttr) + ) + + private def addEventAndReturn[A](events: ListBuffer[Event], event: Event, a: A): IO[A] = + addEventAndReturnIO(events, event, IO.pure(a)) + + private def addEventAndReturnIO[A](events: ListBuffer[Event], event: Event, io: IO[A]): IO[A] = + IO(events.addOne(event)) >> io + +} diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/BatchFilesSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFilesSuite.scala similarity index 95% rename from delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/BatchFilesSpec.scala rename to delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFilesSuite.scala index 49570f2a79..4dfc10f547 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/BatchFilesSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFilesSuite.scala @@ -1,16 +1,16 @@ -package ch.epfl.bluebrain.nexus.delta.plugins.storage.files +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch import cats.effect.IO import cats.implicits.catsSyntaxApplicativeError import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.BatchFilesSpec._ -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch.{BatchCopy, BatchFiles} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch.BatchFilesSuite._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.generators.FileGen import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.mocks.BatchCopyMock import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileCommand.CreateFile import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection.CopyRejection import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileCommand, FileRejection} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.CopyFileSource +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{FetchFileStorage, FileFixtures, FileResource} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StorageFixtures import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.CopyFileRejection.TotalCopySizeTooLarge @@ -25,7 +25,7 @@ import ch.epfl.bluebrain.nexus.testkit.mu.NexusSuite import java.util.UUID import scala.collection.mutable.ListBuffer -class BatchFilesSpec extends NexusSuite with StorageFixtures with Generators with FileFixtures with FileGen { +class BatchFilesSuite extends NexusSuite with StorageFixtures with Generators with FileFixtures with FileGen { test("batch copying should fetch storage, perform copy and evaluate create file commands") { val events = ListBuffer.empty[Event] @@ -106,7 +106,7 @@ class BatchFilesSpec extends NexusSuite with StorageFixtures with Generators wit } } -object BatchFilesSpec { +object BatchFilesSuite { sealed trait Event final case class ActiveStorageFetched( storageIdOpt: Option[IdSegment], diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/generators/FileGen.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/generators/FileGen.scala index b52369f132..d441bc40dd 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/generators/FileGen.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/generators/FileGen.scala @@ -5,7 +5,7 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileCommand.Cre import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{CopyFileDestination, FileAttributes, FileId, FileState} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.CopyFileSource import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{schemas, FileFixtures, FileResource} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StorageGen +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{StorageGen, StorageResource} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{Storage, StorageState, StorageType, StorageValue} import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.nxv @@ -33,7 +33,9 @@ trait FileGen { self: Generators with FileFixtures => def genUser(): User = User(genString(), Label.unsafe(genString())) def genFilesIdsInProject(projRef: ProjectRef): NonEmptyList[FileId] = - NonEmptyList.of(genString(), genString()).map(id => FileId(id, projRef)) + NonEmptyList.of(genFileId(projRef), genFileId(projRef)) + + def genFileId(projRef: ProjectRef) = FileId(genString(), projRef) def genFileIdWithRev(projRef: ProjectRef): FileId = FileId(genString(), 4, projRef) @@ -50,14 +52,34 @@ trait FileGen { self: Generators with FileFixtures => CopyFileDestination(proj, genOption(IdSegment(storage.id.toString)), genOption(genUserTag)) def genUserTag: UserTag = UserTag.unsafe(genString()) def genOption[A](genA: => A): Option[A] = if (Random.nextInt(2) % 2 == 0) Some(genA) else None - def genFileResource(fileId: FileId, context: ProjectContext): FileResource = + + def genFileResource(fileId: FileId, context: ProjectContext): FileResource = + genFileResourceWithStorage(fileId, context, genRevision(), 1L) + + def genFileResourceWithStorage( + fileId: FileId, + context: ProjectContext, + storageRef: ResourceRef.Revision, + fileSize: Long + ): FileResource = genFileResourceWithIri( fileId.id.value.toIri(context.apiMappings, context.base).getOrElse(throw new Exception(s"Bad file $fileId")), fileId.project, - genRevision(), - attributes(genString()) + storageRef, + attributes(genString(), size = fileSize) ) + def genFileResourceAndStorage( + fileId: FileId, + context: ProjectContext, + storageVal: StorageValue, + fileSize: Long = 1L + ): (FileResource, StorageResource) = { + val storageRes = StorageGen.resourceFor(genIri(), fileId.project, storageVal) + val storageRef = ResourceRef.Revision(storageRes.id, storageRes.id, storageRes.rev) + (genFileResourceWithStorage(fileId, context, storageRef, fileSize), storageRes) + } + def genFileResourceWithIri( iri: Iri, projRef: ProjectRef, diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/BatchCopyMock.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/BatchCopyMock.scala index 93702a52f7..eb9d398892 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/BatchCopyMock.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/BatchCopyMock.scala @@ -2,7 +2,7 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.mocks import cats.data.NonEmptyList import cats.effect.IO -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.BatchFilesSpec.{BatchCopyCalled, Event} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch.BatchFilesSuite.{BatchCopyCalled, Event} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch.BatchCopy import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.CopyFileSource diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/DiskCopyMock.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/DiskCopyMock.scala new file mode 100644 index 0000000000..3ad1fbe023 --- /dev/null +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/DiskCopyMock.scala @@ -0,0 +1,17 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.mocks + +import cats.data.NonEmptyList +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.DiskStorage +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.{DiskCopyDetails, DiskStorageCopyFiles} + +object DiskCopyMock { + + def unimplemented: DiskStorageCopyFiles = withMockedCopy((_, _) => IO(???)) + + def withMockedCopy( + copyMock: (DiskStorage, NonEmptyList[DiskCopyDetails]) => IO[NonEmptyList[FileAttributes]] + ): DiskStorageCopyFiles = copyMock(_, _) + +} diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/FetchFileResourceMock.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/FetchFileResourceMock.scala new file mode 100644 index 0000000000..3a4f2d93d9 --- /dev/null +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/FetchFileResourceMock.scala @@ -0,0 +1,13 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.mocks + +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileId +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{FetchFileResource, FileResource} + +object FetchFileResourceMock { + + def unimplemented: FetchFileResource = withMockedFetch(_ => IO(???)) + + def withMockedFetch(fetchMock: FileId => IO[FileResource]): FetchFileResource = fetchMock(_) + +} diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/FetchStorageMock.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/FetchStorageMock.scala new file mode 100644 index 0000000000..cd45218d77 --- /dev/null +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/FetchStorageMock.scala @@ -0,0 +1,17 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.mocks + +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{FetchStorage, StorageResource} +import ch.epfl.bluebrain.nexus.delta.sdk.model.IdSegmentRef +import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef + +object FetchStorageMock { + + def unimplemented: FetchStorage = withMockedFetch((_, _) => IO(???)) + + def withMockedFetch(fetchMock: (IdSegmentRef, ProjectRef) => IO[StorageResource]): FetchStorage = new FetchStorage { + override def fetch(id: IdSegmentRef, project: ProjectRef): IO[StorageResource] = fetchMock(id, project) + override def fetchDefault(project: ProjectRef): IO[StorageResource] = ??? + } + +} diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/RemoteCopyMock.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/RemoteCopyMock.scala new file mode 100644 index 0000000000..ed80532a91 --- /dev/null +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/RemoteCopyMock.scala @@ -0,0 +1,18 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.mocks + +import cats.data.NonEmptyList +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.RemoteDiskStorage +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.RemoteDiskStorageCopyFiles +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.model.RemoteDiskCopyDetails + +object RemoteCopyMock { + + def unimplemented: RemoteDiskStorageCopyFiles = withMockedCopy((_, _) => IO(???)) + + def withMockedCopy( + copyMock: (RemoteDiskStorage, NonEmptyList[RemoteDiskCopyDetails]) => IO[NonEmptyList[FileAttributes]] + ): RemoteDiskStorageCopyFiles = copyMock(_, _) + +} diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/StoragesStatisticsMock.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/StoragesStatisticsMock.scala new file mode 100644 index 0000000000..1b2c918233 --- /dev/null +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/StoragesStatisticsMock.scala @@ -0,0 +1,15 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.mocks + +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StoragesStatistics +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageStatEntry +import ch.epfl.bluebrain.nexus.delta.sdk.model.IdSegment +import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef + +object StoragesStatisticsMock { + + def unimplemented: StoragesStatistics = withMockedGet((_, _) => IO(???)) + + def withMockedGet(getMock: (IdSegment, ProjectRef) => IO[StorageStatEntry]): StoragesStatistics = + (idSegment: IdSegment, project: ProjectRef) => getMock(idSegment, project) +} From 25fe7e4198f378478aa1dccca2884bebde9793df Mon Sep 17 00:00:00 2001 From: dantb Date: Wed, 13 Dec 2023 22:09:25 +0100 Subject: [PATCH 16/18] Tidy up --- .../delta/plugins/storage/files/Files.scala | 8 +-- .../storage/files/batch/BatchCopy.scala | 5 +- .../storage/files/batch/BatchFiles.scala | 2 +- .../storage/files/model/CopyFileDetails.scala | 9 --- .../files/routes/BatchFilesRoutes.scala | 16 +----- .../storage/files/routes/CopyFileSource.scala | 23 ++++---- .../files/routes/CopyFilesResponse.scala | 13 ----- .../storage/files/routes/FilesRoutes.scala | 24 ++++---- .../plugins/storage/storages/Storages.scala | 28 +++++----- .../storages/model/StorageRejection.scala | 2 +- .../operations/StorageFileRejection.scala | 13 +++++ .../disk/DiskStorageCopyFiles.scala | 2 +- .../disk/DiskStorageFetchFile.scala | 4 -- .../operations/disk/DiskStorageSaveFile.scala | 2 +- .../remote/RemoteDiskStorageLinkFile.scala | 30 +++++----- .../client/RemoteDiskStorageClient.scala | 35 ++++-------- .../client/model/RemoteDiskCopyPaths.scala | 9 +++ .../errors/tag-and-rev-copy-error.json | 5 -- .../files/file-bulk-copy-response.json | 8 --- .../plugins/storage/files/FileFixtures.scala | 30 ++-------- .../plugins/storage/files/FilesSpec.scala | 19 ++----- .../storage/files/batch/BatchCopySuite.scala | 5 +- .../storage/files/batch/BatchFilesSuite.scala | 5 +- .../storage/files/generators/FileGen.scala | 31 +++++++++- .../files/routes/FilesRoutesSpec.scala | 36 +++--------- .../rdf/jsonld/encoder/JsonLdEncoder.scala | 2 - .../sdk/directives/ResponseToJsonLd.scala | 42 ++++++-------- .../nexus/delta/sdk/model/ResourceF.scala | 6 -- .../nexus/delta/sdk/utils/RouteFixtures.scala | 3 +- .../docs/delta/api/assets/files/copy-put.json | 34 ----------- .../docs/delta/api/assets/files/copy-put.sh | 10 ---- .../main/paradox/docs/delta/api/files-api.md | 56 ------------------- .../nexus/storage/StorageError.scala | 9 --- .../bluebrain/nexus/tests/HttpClient.scala | 2 +- .../nexus/tests/kg/StorageSpec.scala | 10 ++-- 35 files changed, 170 insertions(+), 368 deletions(-) delete mode 100644 delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/CopyFileDetails.scala delete mode 100644 delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/CopyFilesResponse.scala delete mode 100644 delta/plugins/storage/src/test/resources/errors/tag-and-rev-copy-error.json delete mode 100644 delta/plugins/storage/src/test/resources/files/file-bulk-copy-response.json delete mode 100644 docs/src/main/paradox/docs/delta/api/assets/files/copy-put.json delete mode 100644 docs/src/main/paradox/docs/delta/api/assets/files/copy-put.sh diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala index d0e725f17b..e03e7d7b7e 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala @@ -422,8 +422,8 @@ final class Files( private def test(cmd: FileCommand) = log.dryRun(cmd.project, cmd.id, cmd) - def fetchAndValidateActiveStorage(storageIdOpt: Option[IdSegment], ref: ProjectRef, pc: ProjectContext)(implicit - caller: Caller + override def fetchAndValidateActiveStorage(storageIdOpt: Option[IdSegment], ref: ProjectRef, pc: ProjectContext)( + implicit caller: Caller ): IO[(ResourceRef.Revision, Storage)] = storageIdOpt match { case Some(storageId) => @@ -442,7 +442,7 @@ final class Files( } yield ResourceRef.Revision(storage.id, storage.rev) -> storage.value } - def validateAuth(project: ProjectRef, permission: Permission)(implicit c: Caller): IO[Unit] = + private def validateAuth(project: ProjectRef, permission: Permission)(implicit c: Caller): IO[Unit] = aclCheck.authorizeForOr(project, permission)(AuthorizationFailed(project, permission)) private def extractFileAttributes(iri: Iri, entity: HttpEntity, storage: Storage): IO[FileAttributes] = @@ -467,7 +467,7 @@ final class Files( WrappedStorageRejection(s) } - def generateId(pc: ProjectContext): IO[Iri] = + private def generateId(pc: ProjectContext): IO[Iri] = uuidF().map(uuid => pc.base.iri / uuid.toString) /** diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopy.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopy.scala index 31ff209144..37c94fd763 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopy.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopy.scala @@ -9,10 +9,9 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.CopyFileSource import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.{DiskStorage, RemoteDiskStorage} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageRejection.DifferentStorageType import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{Storage, StorageType} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.CopyFileRejection -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.CopyFileRejection.{SourceFileTooLarge, TotalCopySizeTooLarge} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.CopyFileRejection._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.{DiskCopyDetails, DiskStorageCopyFiles} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.RemoteDiskStorageCopyFiles import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.model.RemoteDiskCopyDetails @@ -106,7 +105,7 @@ object BatchCopy { .getOrElse(differentStorageTypeError(destStorage, sourceStorage)) private def differentStorageTypeError[A](destStorage: Storage, sourceStorage: Storage) = - IO.raiseError[A](DifferentStorageType(sourceStorage.id, found = sourceStorage.tpe, expected = destStorage.tpe)) + IO.raiseError[A](DifferentStorageTypes(sourceStorage.id, sourceStorage.tpe, destStorage.tpe)) private def unsupported(tpe: StorageType) = IO.raiseError(CopyFileRejection.UnsupportedOperation(tpe)) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFiles.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFiles.scala index 99d1d03453..d3098dbc82 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFiles.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFiles.scala @@ -69,7 +69,7 @@ object BatchFiles { } yield resource } - def generateId(pc: ProjectContext): IO[Iri] = + private def generateId(pc: ProjectContext): IO[Iri] = uuidF().map(uuid => pc.base.iri / uuid.toString) private def evalCreateCommand(command: CreateFile) = diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/CopyFileDetails.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/CopyFileDetails.scala deleted file mode 100644 index 8b8b7d72ef..0000000000 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/CopyFileDetails.scala +++ /dev/null @@ -1,9 +0,0 @@ -package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model - -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage - -final case class CopyFileDetails( - destinationDesc: FileDescription, - sourceAttributes: FileAttributes, - sourceStorage: Storage -) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesRoutes.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesRoutes.scala index 2cef55baef..169abeb080 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesRoutes.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesRoutes.scala @@ -27,20 +27,6 @@ import ch.epfl.bluebrain.nexus.delta.sdk.jsonld.BulkOperationResults import ch.epfl.bluebrain.nexus.delta.sdk.model.{BaseUri, IdSegment} import kamon.instrumentation.akka.http.TracingDirectives.operationName -/** - * The files routes - * - * @param identities - * the identity module - * @param aclCheck - * to check acls - * @param files - * the files module - * @param schemeDirectives - * directives related to orgs and projects - * @param index - * the indexing action on write operations - */ final class BatchFilesRoutes( identities: Identities, aclCheck: AclCheck, @@ -96,7 +82,7 @@ final class BatchFilesRoutes( _ <- EitherT.right[FileRejection](logger.info(s"Bulk file copy succeeded with results: $results")) } yield BulkOperationResults(results.toList)) .onError(e => - EitherT.right(logger.error(s"Bulk file copy operation failed for source $source and destination $dest with $e")) + EitherT.right(logger.error(e)(s"Bulk file copy operation failed for source $source and destination $dest")) ) .value } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/CopyFileSource.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/CopyFileSource.scala index 65cf8d8d2b..c259fa68e8 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/CopyFileSource.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/CopyFileSource.scala @@ -20,19 +20,20 @@ object CopyFileSource { sourceFile <- j.hcursor.get[String]("sourceFileId").map(IdSegment(_)) sourceTag <- j.hcursor.get[Option[UserTag]]("sourceTag") sourceRev <- j.hcursor.get[Option[Int]]("sourceRev") - fileId <- (sourceTag, sourceRev) match { - case (Some(tag), None) => Right(FileId(sourceFile, tag, proj)) - case (None, Some(rev)) => Right(FileId(sourceFile, rev, proj)) - case (None, None) => Right(FileId(sourceFile, proj)) - case (Some(_), Some(_)) => - // TODO any decoding failures will return a 415 which isn't accurate most of the time. It should - // probably be a bad request instead. - Left( - DecodingFailure("Tag and revision cannot be simultaneously present for source file lookup", Nil) - ) - } + fileId <- parseFileId(sourceFile, proj, sourceTag, sourceRev) } yield fileId + def parseFileId(id: IdSegment, proj: ProjectRef, sourceTag: Option[UserTag], sourceRev: Option[Int]) = + (sourceTag, sourceRev) match { + case (Some(tag), None) => Right(FileId(id, tag, proj)) + case (None, Some(rev)) => Right(FileId(id, rev, proj)) + case (None, None) => Right(FileId(id, proj)) + case (Some(_), Some(_)) => + Left( + DecodingFailure("Tag and revision cannot be simultaneously present for source file lookup", Nil) + ) + } + for { sourceProj <- cur.get[ProjectRef]("sourceProjectRef") files <- cur.get[NonEmptyList[Json]]("files").flatMap(_.traverse(parseSingle(_, sourceProj))) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/CopyFilesResponse.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/CopyFilesResponse.scala deleted file mode 100644 index 1e801ba2cc..0000000000 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/CopyFilesResponse.scala +++ /dev/null @@ -1,13 +0,0 @@ -package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes - -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.FileResource - -final case class FailureSummary( - source: CopyFileSource, - reason: String // TODO use ADT -) - -final case class CopyFilesResponse( - successes: List[FileResource], - failures: List[FailureSummary] -) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutes.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutes.scala index 358fcab7b9..815ca13d34 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutes.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutes.scala @@ -73,9 +73,9 @@ final class FilesRoutes( (baseUriPrefix(baseUri.prefix) & replaceUri("files", schemas.files)) { pathPrefix("files") { extractCaller { implicit caller => - resolveProjectRef.apply { projectRef => + resolveProjectRef.apply { ref => implicit class IndexOps(io: IO[FileResource]) { - def index(m: IndexingMode): IO[FileResource] = io.flatTap(self.index(projectRef, _, m)) + def index(m: IndexingMode): IO[FileResource] = io.flatTap(self.index(ref, _, m)) } concat( @@ -89,7 +89,7 @@ final class FilesRoutes( emit( Created, files - .createLink(storage, projectRef, filename, mediaType, path, tag) + .createLink(storage, ref, filename, mediaType, path, tag) .index(mode) .attemptNarrow[FileRejection] ) @@ -98,14 +98,14 @@ final class FilesRoutes( extractRequestEntity { entity => emit( Created, - files.create(storage, projectRef, entity, tag).index(mode).attemptNarrow[FileRejection] + files.create(storage, ref, entity, tag).index(mode).attemptNarrow[FileRejection] ) } ) } }, (idSegment & indexingMode) { (id, mode) => - val fileId = FileId(id, projectRef) + val fileId = FileId(id, ref) concat( pathEndOrSingleSlash { operationName(s"$prefixSegment/files/{org}/{project}/{id}") { @@ -163,7 +163,7 @@ final class FilesRoutes( }, // Deprecate a file (delete & parameter("rev".as[Int])) { rev => - authorizeFor(projectRef, Write).apply { + authorizeFor(ref, Write).apply { emit( files .deprecate(fileId, rev) @@ -176,7 +176,7 @@ final class FilesRoutes( // Fetch a file (get & idSegmentRef(id)) { id => - emitOrFusionRedirect(projectRef, id, fetch(FileId(id, projectRef))) + emitOrFusionRedirect(ref, id, fetch(FileId(id, ref))) } ) } @@ -185,9 +185,9 @@ final class FilesRoutes( operationName(s"$prefixSegment/files/{org}/{project}/{id}/tags") { concat( // Fetch a file tags - (get & idSegmentRef(id) & pathEndOrSingleSlash & authorizeFor(projectRef, Read)) { id => + (get & idSegmentRef(id) & pathEndOrSingleSlash & authorizeFor(ref, Read)) { id => emit( - fetchMetadata(FileId(id, projectRef)) + fetchMetadata(FileId(id, ref)) .map(_.value.tags) .attemptNarrow[FileRejection] .rejectOn[FileNotFound] @@ -195,7 +195,7 @@ final class FilesRoutes( }, // Tag a file (post & parameter("rev".as[Int]) & pathEndOrSingleSlash) { rev => - authorizeFor(projectRef, Write).apply { + authorizeFor(ref, Write).apply { entity(as[Tag]) { case Tag(tagRev, tag) => emit( Created, @@ -206,7 +206,7 @@ final class FilesRoutes( }, // Delete a tag (tagLabel & delete & parameter("rev".as[Int]) & pathEndOrSingleSlash & authorizeFor( - projectRef, + ref, Write )) { (tag, rev) => emit( @@ -221,7 +221,7 @@ final class FilesRoutes( } }, (pathPrefix("undeprecate") & put & parameter("rev".as[Int])) { rev => - authorizeFor(projectRef, Write).apply { + authorizeFor(ref, Write).apply { emit( files .undeprecate(fileId, rev) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/Storages.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/Storages.scala index 5230d5ef3f..b1ecf85514 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/Storages.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/Storages.scala @@ -234,22 +234,22 @@ final class Storages private ( }.span("deprecateStorage") /** - * Undeprecate a storage - * - * @param id - * the storage identifier to expand as the id of the storage - * @param projectRef - * the project where the storage belongs - * @param rev - * the current revision of the storage - */ + * Undeprecate a storage + * + * @param id + * the storage identifier to expand as the id of the storage + * @param projectRef + * the project where the storage belongs + * @param rev + * the current revision of the storage + */ def undeprecate( - id: IdSegment, - projectRef: ProjectRef, - rev: Int - )(implicit subject: Subject): IO[StorageResource] = { + id: IdSegment, + projectRef: ProjectRef, + rev: Int + )(implicit subject: Subject): IO[StorageResource] = { for { - pc <- fetchContext.onModify(projectRef) + pc <- fetchContext.onModify(projectRef) iri <- expandIri(id, pc) res <- eval(UndeprecateStorage(iri, projectRef, rev, subject)) } yield res diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/StorageRejection.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/StorageRejection.scala index 1e1fccaeea..271d0672bd 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/StorageRejection.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/StorageRejection.scala @@ -143,7 +143,7 @@ object StorageRejection { extends StorageRejection(s"Storage ${id.fold("")(id => s"'$id'")} has invalid JSON-LD payload.") /** - * Signals an attempt to update/create a storage based on a previous revision with a different storage type + * Signals an attempt to update a storage to a different storage type * * @param id * @param found diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/StorageFileRejection.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/StorageFileRejection.scala index 69dd3023d3..d22c67bcbb 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/StorageFileRejection.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/StorageFileRejection.scala @@ -4,6 +4,7 @@ import akka.http.scaladsl.model.StatusCodes import ch.epfl.bluebrain.nexus.delta.kernel.error.Rejection import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageType import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri +import ch.epfl.bluebrain.nexus.delta.sdk.http.HttpClientError import ch.epfl.bluebrain.nexus.delta.sdk.marshalling.HttpResponseFields /** @@ -89,10 +90,22 @@ object StorageFileRejection { s"Combined size of source files ($totalSize) exceeds space ($spaceLeft) on destination storage $storageId" ) + final case class RemoteDiskClientError(underlying: HttpClientError) + extends CopyFileRejection( + s"Error from remote disk storage client: ${underlying.asString}" + ) + + final case class DifferentStorageTypes(id: Iri, source: StorageType, dest: StorageType) + extends CopyFileRejection( + s"Source storage $id of type $source cannot be different to the destination storage type $dest" + ) + implicit val statusCodes: HttpResponseFields[CopyFileRejection] = HttpResponseFields { case _: UnsupportedOperation => StatusCodes.BadRequest case _: SourceFileTooLarge => StatusCodes.BadRequest case _: TotalCopySizeTooLarge => StatusCodes.BadRequest + case _: DifferentStorageTypes => StatusCodes.BadRequest + case _: RemoteDiskClientError => StatusCodes.InternalServerError } } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageCopyFiles.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageCopyFiles.scala index 75f95d6a9a..8218d4b715 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageCopyFiles.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageCopyFiles.scala @@ -37,7 +37,7 @@ object DiskStorageCopyFiles { } } yield (copyDetails, destAttr) - private def computeDestLocation(destStorage: DiskStorage, cd: DiskCopyDetails): IO[(file.Path, file.Path)] = + private def computeDestLocation(destStorage: DiskStorage, cd: DiskCopyDetails) = computeLocation(destStorage.project, destStorage.value, cd.destinationDesc.uuid, cd.destinationDesc.filename) private def mkDestAttributes(cd: DiskCopyDetails, destPath: file.Path, destRelativePath: file.Path) = diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageFetchFile.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageFetchFile.scala index 7068ea94fa..2bc3943790 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageFetchFile.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageFetchFile.scala @@ -9,10 +9,6 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.Storage import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.FetchFileRejection.UnexpectedLocationFormat import ch.epfl.bluebrain.nexus.delta.sdk.AkkaSource -import java.net.URI -import java.nio.file.Paths -import scala.util.{Failure, Success, Try} - object DiskStorageFetchFile extends FetchFile { override def apply(attributes: FileAttributes): IO[AkkaSource] = diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageSaveFile.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageSaveFile.scala index ef428d3d27..2172233418 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageSaveFile.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageSaveFile.scala @@ -74,7 +74,7 @@ object DiskStorageSaveFile { for { (resolved, relative) <- computeLocation(project, disk, uuid, filename) dir = resolved.getParent - _ <- IO.delay(Files.createDirectories(dir)).adaptError(couldNotCreateDirectory(dir, _)) + _ <- IO.blocking(Files.createDirectories(dir)).adaptError(couldNotCreateDirectory(dir, _)) } yield resolved -> relative def computeLocation( diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageLinkFile.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageLinkFile.scala index 2c4cd47f7c..087d6c8436 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageLinkFile.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageLinkFile.scala @@ -2,7 +2,6 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote import akka.http.scaladsl.model.Uri import cats.effect.IO -import ch.epfl.bluebrain.nexus.delta.kernel.Logger import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes.FileAttributesOrigin.Storage import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes, FileDescription} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.RemoteDiskStorage @@ -13,24 +12,21 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote. class RemoteDiskStorageLinkFile(storage: RemoteDiskStorage, client: RemoteDiskStorageClient) extends LinkFile { - private val logger = Logger[RemoteDiskStorageLinkFile] - def apply(sourcePath: Uri.Path, description: FileDescription): IO[FileAttributes] = { val destinationPath = Uri.Path(intermediateFolders(storage.project, description.uuid, description.filename)) - logger.info(s"DTBDTB doing link file with ${storage.value.folder}, source $sourcePath, dest $destinationPath") >> - client.moveFile(storage.value.folder, sourcePath, destinationPath)(storage.value.endpoint).map { - case RemoteDiskStorageFileAttributes(location, bytes, digest, _) => - FileAttributes( - uuid = description.uuid, - location = location, - path = destinationPath, - filename = description.filename, - mediaType = description.mediaType, - bytes = bytes, - digest = digest, - origin = Storage - ) - } + client.moveFile(storage.value.folder, sourcePath, destinationPath)(storage.value.endpoint).map { + case RemoteDiskStorageFileAttributes(location, bytes, digest, _) => + FileAttributes( + uuid = description.uuid, + location = location, + path = destinationPath, + filename = description.filename, + mediaType = description.mediaType, + bytes = bytes, + digest = digest, + origin = Storage + ) + } } } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/RemoteDiskStorageClient.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/RemoteDiskStorageClient.scala index 1eb2a9b84a..5aac781cfb 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/RemoteDiskStorageClient.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/RemoteDiskStorageClient.scala @@ -12,7 +12,7 @@ import cats.effect.IO import cats.implicits._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.FetchFileRejection.UnexpectedFetchError import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.MoveFileRejection.UnexpectedMoveError -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.{FetchFileRejection, MoveFileRejection, SaveFileRejection} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.{CopyFileRejection, FetchFileRejection, MoveFileRejection, SaveFileRejection} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.model.{RemoteDiskCopyPaths, RemoteDiskStorageFileAttributes} import ch.epfl.bluebrain.nexus.delta.rdf.implicits.uriDecoder import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.JsonLdContext.keywords @@ -178,43 +178,28 @@ final class RemoteDiskStorageClient(client: HttpClient, getAuthToken: AuthTokenP } /** - * Moves a path from the provided ''sourceRelativePath'' to ''destRelativePath'' inside the nexus folder. + * Copies files to a destination bucket. Source files can be located within different buckets. File attributes are + * not recomputed, so only the file paths will change. * - * @param destBucket - * the storage bucket name - * @param sourceRelativePath - * the source relative path location - * @param destRelativePath - * the destination relative path location inside the nexus folder + * If any copies fail the whole operation will be aborted and the remote storage service will return an error. + * + * @return + * Absolute locations of the created files, preserving the input order. */ def copyFiles( destBucket: Label, files: NonEmptyList[RemoteDiskCopyPaths] - )(implicit baseUri: BaseUri): IO[NonEmptyList[Uri]] = { + )(implicit baseUri: BaseUri): IO[NonEmptyList[Uri]] = getAuthToken(credentials).flatMap { authToken => val endpoint = baseUri.endpoint / "buckets" / destBucket.value / "files" - val payload = files.map { case RemoteDiskCopyPaths(sourceBucket, source, dest) => - Json.obj("sourceBucket" := sourceBucket, "source" := source.toString(), "destination" := dest.toString()) - }.asJson implicit val dec: Decoder[NonEmptyList[Uri]] = Decoder[NonEmptyList[Json]].emap { nel => nel.traverse(_.hcursor.get[Uri]("absoluteDestinationLocation").leftMap(_.toString())) } client - .fromJsonTo[NonEmptyList[Uri]](Post(endpoint, payload).withCredentials(authToken)) - // TODO update error -// .adaptError { -// case error @ HttpClientStatusError(_, `NotFound`, _) if !bucketNotFoundType(error) => -// MoveFileRejection.FileNotFound(sourceRelativePath.toString) -// case error @ HttpClientStatusError(_, `BadRequest`, _) if pathContainsLinksType(error) => -// MoveFileRejection.PathContainsLinks(destRelativePath.toString) -// case HttpClientStatusError(_, `Conflict`, _) => -// MoveFileRejection.ResourceAlreadyExists(destRelativePath.toString) -// case error: HttpClientError => -// UnexpectedMoveError(sourceRelativePath.toString, destRelativePath.toString, error.asString) -// } + .fromJsonTo[NonEmptyList[Uri]](Post(endpoint, files.asJson).withCredentials(authToken)) + .adaptError { case error: HttpClientError => CopyFileRejection.RemoteDiskClientError(error) } } - } private def bucketNotFoundType(error: HttpClientError): Boolean = error.jsonBody.fold(false)(_.hcursor.get[String](keywords.tpe).toOption.contains("BucketNotFound")) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/model/RemoteDiskCopyPaths.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/model/RemoteDiskCopyPaths.scala index fdff1e945e..c19726b430 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/model/RemoteDiskCopyPaths.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/model/RemoteDiskCopyPaths.scala @@ -2,9 +2,18 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote import akka.http.scaladsl.model.Uri.Path import ch.epfl.bluebrain.nexus.delta.sourcing.model.Label +import io.circe.syntax.KeyOps +import io.circe.{Encoder, Json} final case class RemoteDiskCopyPaths( sourceBucket: Label, sourcePath: Path, destPath: Path ) + +object RemoteDiskCopyPaths { + implicit val enc: Encoder[RemoteDiskCopyPaths] = Encoder.instance { + case RemoteDiskCopyPaths(sourceBucket, source, dest) => + Json.obj("sourceBucket" := sourceBucket, "source" := source.toString(), "destination" := dest.toString()) + } +} diff --git a/delta/plugins/storage/src/test/resources/errors/tag-and-rev-copy-error.json b/delta/plugins/storage/src/test/resources/errors/tag-and-rev-copy-error.json deleted file mode 100644 index 111259dff1..0000000000 --- a/delta/plugins/storage/src/test/resources/errors/tag-and-rev-copy-error.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "@context" : "https://bluebrain.github.io/nexus/contexts/error.json", - "@type" : "InvalidFileLookup", - "reason" : "Only one of 'tag' and 'rev' can be used to lookup file '{{fileId}}'." -} diff --git a/delta/plugins/storage/src/test/resources/files/file-bulk-copy-response.json b/delta/plugins/storage/src/test/resources/files/file-bulk-copy-response.json deleted file mode 100644 index 82e8e03eee..0000000000 --- a/delta/plugins/storage/src/test/resources/files/file-bulk-copy-response.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "@context": [ - "https://bluebrain.github.io/nexus/contexts/bulk-operation.json", - "https://bluebrain.github.io/nexus/contexts/metadata.json", - "https://bluebrain.github.io/nexus/contexts/files.json" - ], - "_results": {{results}} -} \ No newline at end of file diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FileFixtures.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FileFixtures.scala index cf34480c00..57ab705e79 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FileFixtures.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FileFixtures.scala @@ -1,39 +1,31 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.files import akka.http.scaladsl.model.ContentTypes.`text/plain(UTF-8)` -import akka.http.scaladsl.model.{HttpEntity, MessageEntity, Multipart, Uri} +import akka.http.scaladsl.model.{HttpEntity, MessageEntity, Multipart} import cats.effect.unsafe.implicits.global import cats.effect.{IO, Ref} import ch.epfl.bluebrain.nexus.delta.kernel.utils.{UUIDF, UrlUtils} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest.ComputedDigest +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.generators.FileGen import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes.FileAttributesOrigin.Client -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{AbsolutePath, DigestAlgorithm} import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.nxv import ch.epfl.bluebrain.nexus.delta.sdk.generators.ProjectGen import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.ApiMappings import ch.epfl.bluebrain.nexus.delta.sourcing.model.{Label, ProjectRef} -import java.nio.file.{Files => JavaFiles} import java.util.{Base64, UUID} trait FileFixtures { val uuid = UUID.fromString("8249ba90-7cc6-4de5-93a1-802c04200dcc") val uuid2 = UUID.fromString("12345678-7cc6-4de5-93a1-802c04200dcc") - val uuid3 = UUID.randomUUID() - val uuid4 = UUID.randomUUID() val uuidOrg2 = UUID.fromString("66666666-7cc6-4de5-93a1-802c04200dcc") val ref = Ref.of[IO, UUID](uuid).unsafeRunSync() implicit val uuidF: UUIDF = UUIDF.fromRef(ref) val org = Label.unsafe("org") val org2 = Label.unsafe("org2") val project = ProjectGen.project(org.value, "proj", base = nxv.base, mappings = ApiMappings("file" -> schemas.files)) - val project2 = - ProjectGen.project(org2.value, "proj2", base = nxv.base, mappings = ApiMappings("file" -> schemas.files)) val deprecatedProject = ProjectGen.project("org", "proj-deprecated") val projectRef = project.ref - val projectRefOrg2 = project2.ref val diskId2 = nxv + "disk2" val file1 = nxv + "file1" val file2 = nxv + "file2" @@ -45,9 +37,7 @@ trait FileFixtures { val generatedId2 = project.base.iri / uuid2.toString val content = "file content" - val path = AbsolutePath(JavaFiles.createTempDirectory("files")).fold(e => throw new Exception(e), identity) - val digest = - ComputedDigest(DigestAlgorithm.default, "e0ac3601005dfa1864f5392aabaf7d898b1b5bab854f1acb4491bcd806b76b0c") + val path = FileGen.path def withUUIDF[T](id: UUID)(test: => T): T = (for { old <- ref.getAndSet(id) @@ -60,19 +50,7 @@ trait FileFixtures { size: Long = 12, id: UUID = uuid, projRef: ProjectRef = projectRef - ): FileAttributes = { - val uuidPathSegment = id.toString.take(8).mkString("/") - FileAttributes( - id, - s"file://$path/${projRef.toString}/$uuidPathSegment/$filename", - Uri.Path(s"${projRef.toString}/$uuidPathSegment/$filename"), - filename, - Some(`text/plain(UTF-8)`), - size, - digest, - Client - ) - } + ): FileAttributes = FileGen.attributes(filename, size, id, projRef) def entity(filename: String = "file.txt"): MessageEntity = Multipart diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala index e5757b214c..d9eebc1bae 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala @@ -88,20 +88,17 @@ class FilesSpec(docker: RemoteStorageDocker) val diskId: IdSegment = nxv + "disk" val diskRev = ResourceRef.Revision(iri"$diskId?rev=1", diskIdIri, 1) - val smallDiskId: IdSegment = nxv + "smalldisk" - val storageIri = nxv + "other-storage" val storage: IdSegment = nxv + "other-storage" val fetchContext = FetchContextDummy( - Map(project.ref -> project.context, project2.ref -> project2.context), + Map(project.ref -> project.context), Set(deprecatedProject.ref) ) val aclCheck = AclSimpleCheck( (Anonymous, AclAddress.Root, Set(Permissions.resources.read)), (bob, AclAddress.Project(projectRef), Set(diskFields.readPermission.value, diskFields.writePermission.value)), - (bob, AclAddress.Project(projectRefOrg2), Set(diskFields.readPermission.value, diskFields.writePermission.value)), (alice, AclAddress.Project(projectRef), Set(otherRead, otherWrite)) ).accepted @@ -110,10 +107,8 @@ class FilesSpec(docker: RemoteStorageDocker) remoteDisk = Some(config.remoteDisk.value.copy(defaultMaxFileSize = 500)) ) - val storageStatistics: StoragesStatistics = { - case (`smallDiskId`, _) => IO.pure { StorageStatEntry(10L, 0L) } - case (_, _) => IO.pure { StorageStatEntry(10L, 100L) } - } + val storageStatistics: StoragesStatistics = + (_, _) => IO.pure { StorageStatEntry(10L, 100L) } lazy val storages: Storages = Storages( fetchContext.mapRejection(StorageRejection.ProjectContextRejection), @@ -158,12 +153,10 @@ class FilesSpec(docker: RemoteStorageDocker) "create storages for files" in { val payload = diskFieldsJson deepMerge json"""{"capacity": 320, "maxFileSize": 300, "volume": "$path"}""" storages.create(diskId, projectRef, payload).accepted - storages.create(diskId, projectRefOrg2, payload).accepted val payload2 = json"""{"@type": "RemoteDiskStorage", "endpoint": "${docker.hostConfig.endpoint}", "folder": "${RemoteStorageDocker.BucketName}", "readPermission": "$otherRead", "writePermission": "$otherWrite", "maxFileSize": 300, "default": false}""" storages.create(remoteId, projectRef, payload2).accepted - storages.create(remoteId, projectRefOrg2, payload2).accepted } "succeed with the id passed" in { @@ -630,12 +623,10 @@ class FilesSpec(docker: RemoteStorageDocker) } - def givenAFile(assertion: FileId => Assertion): Assertion = givenAFileWithSize(1)(assertion) - - def givenAFileWithSize(size: Int)(assertion: FileId => Assertion): Assertion = { + def givenAFile(assertion: FileId => Assertion): Assertion = { val filename = genString() val id = fileId(filename) - files.create(id, Some(diskId), randomEntity(filename, size), None).accepted + files.create(id, Some(diskId), randomEntity(filename, 1), None).accepted files.fetch(id).accepted assertion(id) } diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopySuite.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopySuite.scala index 525c273cd8..014ad46d98 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopySuite.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopySuite.scala @@ -11,9 +11,8 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.CopyFileSource import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{FetchFileResource, FileFixtures, FileResource} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.{DiskStorage, RemoteDiskStorage, S3Storage} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageRejection.DifferentStorageType import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{Storage, StorageStatEntry, StorageType} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.CopyFileRejection.{SourceFileTooLarge, TotalCopySizeTooLarge, UnsupportedOperation} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.CopyFileRejection.{DifferentStorageTypes, SourceFileTooLarge, TotalCopySizeTooLarge, UnsupportedOperation} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.{DiskCopyDetails, DiskStorageCopyFiles} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.RemoteDiskStorageCopyFiles import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.model.RemoteDiskCopyDetails @@ -107,7 +106,7 @@ class BatchCopySuite extends NexusSuite with StorageFixtures with Generators wit fetchStorage = stubbedFetchStorage(sourceStorage, events), aclCheck = aclCheck ) - val expectedError = DifferentStorageType(sourceStorage.id, StorageType.DiskStorage, StorageType.RemoteDiskStorage) + val expectedError = DifferentStorageTypes(sourceStorage.id, StorageType.DiskStorage, StorageType.RemoteDiskStorage) batchCopy.copyFiles(source, genRemoteStorage())(caller(user)).interceptEquals(expectedError).map { _ => val obtainedEvents = events.toList diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFilesSuite.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFilesSuite.scala index 4dfc10f547..4343fec92a 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFilesSuite.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFilesSuite.scala @@ -1,7 +1,6 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch import cats.effect.IO -import cats.implicits.catsSyntaxApplicativeError import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch.BatchFilesSuite._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.generators.FileGen @@ -69,9 +68,9 @@ class BatchFilesSuite extends NexusSuite with StorageFixtures with Generators wi val sourceProj = genProject() val (source, destination) = (genCopyFileSource(sourceProj.ref), genCopyFileDestination(destProj.ref, destStorage.storage)) - val result = batchFiles.copyFiles(source, destination).attemptNarrow[CopyRejection].accepted + val expectedError = CopyRejection(sourceProj.ref, destProj.ref, destStorage.id, error) - assertEquals(result, Left(CopyRejection(sourceProj.ref, destProj.ref, destStorage.id, error))) + batchFiles.copyFiles(source, destination).interceptEquals(expectedError).accepted val expectedActiveStorageFetched = ActiveStorageFetched(destination.storage, destProj.ref, destProj.context, c) val expectedBatchCopyCalled = BatchCopyCalled(source, destStorage.storage, c) diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/generators/FileGen.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/generators/FileGen.scala index d441bc40dd..41e0a7c9b9 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/generators/FileGen.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/generators/FileGen.scala @@ -1,12 +1,16 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.generators +import akka.http.scaladsl.model.ContentTypes.`text/plain(UTF-8)` +import akka.http.scaladsl.model.Uri import cats.data.NonEmptyList +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest.ComputedDigest +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes.FileAttributesOrigin.Client import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileCommand.CreateFile import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{CopyFileDestination, FileAttributes, FileId, FileState} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.CopyFileSource import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{schemas, FileFixtures, FileResource} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{StorageGen, StorageResource} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{Storage, StorageState, StorageType, StorageValue} import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.nxv import ch.epfl.bluebrain.nexus.delta.sdk.generators.ProjectGen @@ -17,7 +21,9 @@ import ch.epfl.bluebrain.nexus.delta.sourcing.model.Tag.UserTag import ch.epfl.bluebrain.nexus.delta.sourcing.model.{Label, ProjectRef, ResourceRef} import ch.epfl.bluebrain.nexus.testkit.Generators +import java.nio.file.{Files => JavaFiles} import java.time.Instant +import java.util.UUID import scala.util.Random trait FileGen { self: Generators with FileFixtures => @@ -143,4 +149,27 @@ object FileGen { ): FileResource = state(id, project, storage, attributes, storageType, rev, deprecated, tags, createdBy, updatedBy).toResource + lazy val path = AbsolutePath(JavaFiles.createTempDirectory("files")).fold(e => throw new Exception(e), identity) + + private val digest = + ComputedDigest(DigestAlgorithm.default, "e0ac3601005dfa1864f5392aabaf7d898b1b5bab854f1acb4491bcd806b76b0c") + + def attributes( + filename: String, + size: Long, + id: UUID, + projRef: ProjectRef + ): FileAttributes = { + val uuidPathSegment = id.toString.take(8).mkString("/") + FileAttributes( + id, + s"file://$path/${projRef.toString}/$uuidPathSegment/$filename", + Uri.Path(s"${projRef.toString}/$uuidPathSegment/$filename"), + filename, + Some(`text/plain(UTF-8)`), + size, + digest, + Client + ) + } } diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala index 901aefa286..ffe77348d5 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala @@ -43,7 +43,6 @@ import ch.epfl.bluebrain.nexus.testkit.ce.IOFromMap import ch.epfl.bluebrain.nexus.testkit.errors.files.FileErrors.{fileAlreadyExistsError, fileIsNotDeprecatedError} import ch.epfl.bluebrain.nexus.testkit.scalatest.ce.CatsIOValues import io.circe.Json -import io.circe.syntax.EncoderOps import org.scalatest._ class FilesRoutesSpec @@ -64,14 +63,13 @@ class FilesRoutesSpec // TODO: sort out how we handle this in tests implicit override def rcr: RemoteContextResolution = RemoteContextResolution.fixedIO( - storageContexts.storages -> ContextValue.fromFile("contexts/storages.json"), - storageContexts.storagesMetadata -> ContextValue.fromFile("contexts/storages-metadata.json"), - fileContexts.files -> ContextValue.fromFile("contexts/files.json"), - Vocabulary.contexts.metadata -> ContextValue.fromFile("contexts/metadata.json"), - Vocabulary.contexts.error -> ContextValue.fromFile("contexts/error.json"), - Vocabulary.contexts.tags -> ContextValue.fromFile("contexts/tags.json"), - Vocabulary.contexts.search -> ContextValue.fromFile("contexts/search.json"), - Vocabulary.contexts.bulkOperation -> ContextValue.fromFile("contexts/bulk-operation.json") + storageContexts.storages -> ContextValue.fromFile("contexts/storages.json"), + storageContexts.storagesMetadata -> ContextValue.fromFile("contexts/storages-metadata.json"), + fileContexts.files -> ContextValue.fromFile("contexts/files.json"), + Vocabulary.contexts.metadata -> ContextValue.fromFile("contexts/metadata.json"), + Vocabulary.contexts.error -> ContextValue.fromFile("contexts/error.json"), + Vocabulary.contexts.tags -> ContextValue.fromFile("contexts/tags.json"), + Vocabulary.contexts.search -> ContextValue.fromFile("contexts/search.json") ) private val reader = User("reader", realm) @@ -90,7 +88,7 @@ class FilesRoutesSpec private val asWriter = addCredentials(OAuth2BearerToken("writer")) private val asS3Writer = addCredentials(OAuth2BearerToken("s3writer")) - private val fetchContext = FetchContextDummy(Map(project.ref -> project.context, project2.ref -> project2.context)) + private val fetchContext = FetchContextDummy(Map(project.ref -> project.context)) private val s3Read = Permission.unsafe("s3/read") private val s3Write = Permission.unsafe("s3/write") @@ -139,11 +137,7 @@ class FilesRoutesSpec clock )(uuidF, typedSystem) private val groupDirectives = - DeltaSchemeDirectives( - fetchContext, - ioFromMap(uuid -> projectRef.organization, uuidOrg2 -> projectRefOrg2.organization), - ioFromMap(uuid -> projectRef, uuidOrg2 -> projectRefOrg2) - ) + DeltaSchemeDirectives(fetchContext, ioFromMap(uuid -> projectRef.organization), ioFromMap(uuid -> projectRef)) private lazy val routes = routesWithIdentities(identities) private def routesWithIdentities(identities: Identities) = Route.seal(FilesRoutes(stCfg, identities, aclCheck, files, groupDirectives, IndexingAction.noop)) @@ -171,12 +165,6 @@ class FilesRoutesSpec .create(dId, projectRef, diskFieldsJson deepMerge defaults deepMerge json"""{"capacity":5000}""")(callerWriter) .void .accepted - storages - .create(dId, projectRefOrg2, diskFieldsJson deepMerge defaults deepMerge json"""{"capacity":5000}""")( - callerWriter - ) - .void - .accepted } "File routes" should { @@ -669,9 +657,6 @@ class FilesRoutesSpec test(id) } - def bulkOperationResponse(total: Int, results: List[Json]): Json = - FilesRoutesSpec.bulkOperationResponse(total, results.map(_.removeKeys("@context"))).accepted - def fileMetadata( project: ProjectRef, id: Iri, @@ -725,7 +710,4 @@ object FilesRoutesSpec { "type" -> storageType, "self" -> ResourceUris("files", project, id).accessUri ) - - def bulkOperationResponse(total: Int, results: List[Json]): IO[Json] = - loader.jsonContentOf("files/file-bulk-copy-response.json", "total" -> total, "results" -> results.asJson) } diff --git a/delta/rdf/src/main/scala/ch/epfl/bluebrain/nexus/delta/rdf/jsonld/encoder/JsonLdEncoder.scala b/delta/rdf/src/main/scala/ch/epfl/bluebrain/nexus/delta/rdf/jsonld/encoder/JsonLdEncoder.scala index b6ab2aefbe..786fbb7390 100644 --- a/delta/rdf/src/main/scala/ch/epfl/bluebrain/nexus/delta/rdf/jsonld/encoder/JsonLdEncoder.scala +++ b/delta/rdf/src/main/scala/ch/epfl/bluebrain/nexus/delta/rdf/jsonld/encoder/JsonLdEncoder.scala @@ -97,8 +97,6 @@ trait JsonLdEncoder[A] { object JsonLdEncoder { - def apply[A](enc: JsonLdEncoder[A]): JsonLdEncoder[A] = enc - private def randomRootNode[A]: A => BNode = (_: A) => BNode.random /** diff --git a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/directives/ResponseToJsonLd.scala b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/directives/ResponseToJsonLd.scala index b7bb3926b6..33a3823ea9 100644 --- a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/directives/ResponseToJsonLd.scala +++ b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/directives/ResponseToJsonLd.scala @@ -10,13 +10,12 @@ import akka.http.scaladsl.server.Route import cats.effect.IO import cats.effect.unsafe.implicits._ import cats.syntax.all._ -import ch.epfl.bluebrain.nexus.delta.kernel.Logger import ch.epfl.bluebrain.nexus.delta.rdf.RdfMediaTypes._ import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.{JsonLdApi, JsonLdJavaApi} import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.RemoteContextResolution import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.encoder.JsonLdEncoder import ch.epfl.bluebrain.nexus.delta.rdf.utils.JsonKeyOrdering -import ch.epfl.bluebrain.nexus.delta.sdk.{AkkaSource, JsonLdValue} +import ch.epfl.bluebrain.nexus.delta.sdk.JsonLdValue import ch.epfl.bluebrain.nexus.delta.sdk.directives.ResponseToJsonLd.{RouteOutcome, UseLeft, UseRight} import ch.epfl.bluebrain.nexus.delta.sdk.directives.DeltaDirectives.{emit, jsonLdFormatOrReject, mediaTypes, requestMediaType, unacceptedMediaTypeRejection} import ch.epfl.bluebrain.nexus.delta.sdk.directives.Response.{Complete, Reject} @@ -115,38 +114,31 @@ object ResponseToJsonLd extends FileBytesInstances { val encodedFilename = Base64.getEncoder.encodeToString(filename.getBytes(StandardCharsets.UTF_8)) s"=?UTF-8?B?$encodedFilename?=" } - private val logger = Logger[ResponseToJsonLd] + override def apply(statusOverride: Option[StatusCode]): Route = { val flattened = io.flatMap { _.traverse { fr => - logger.info(s"DTBDTB in file response JSON LD encoding for ${fr.metadata.filename}") >> - fr.content.map { - _.map { s => - fr.metadata -> s - } + fr.content.map { + _.map { s => + fr.metadata -> s } + } } } onSuccess(flattened.unsafeToFuture()) { - thing: Either[Response[E], Either[Complete[JsonLdValue], (FileResponse.Metadata, AkkaSource)]] => - logger.info(s"DTBDTB result of encoding was $thing").unsafeRunSync() - thing match { - case Left(complete: Complete[E]) => emit(complete) - case Left(reject: Reject[E]) => emit(reject) - case Right(Left(c)) => emit(c) - case Right(Right((metadata, content))) => - headerValueByType(Accept) { accept => - if (accept.mediaRanges.exists(_.matches(metadata.contentType.mediaType))) { - val encodedFilename = attachmentString(metadata.filename) - respondWithHeaders( - RawHeader("Content-Disposition", s"""attachment; filename="$encodedFilename"""") - ) { - complete(statusOverride.getOrElse(OK), HttpEntity(metadata.contentType, content)) - } - } else - reject(unacceptedMediaTypeRejection(Seq(metadata.contentType.mediaType))) + case Left(complete: Complete[E]) => emit(complete) + case Left(reject: Reject[E]) => emit(reject) + case Right(Left(c)) => emit(c) + case Right(Right((metadata, content))) => + headerValueByType(Accept) { accept => + if (accept.mediaRanges.exists(_.matches(metadata.contentType.mediaType))) { + val encodedFilename = attachmentString(metadata.filename) + respondWithHeaders(RawHeader("Content-Disposition", s"""attachment; filename="$encodedFilename"""")) { + complete(statusOverride.getOrElse(OK), HttpEntity(metadata.contentType, content)) } + } else + reject(unacceptedMediaTypeRejection(Seq(metadata.contentType.mediaType))) } } } diff --git a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/model/ResourceF.scala b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/model/ResourceF.scala index 66884ecc95..369cee2486 100644 --- a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/model/ResourceF.scala +++ b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/model/ResourceF.scala @@ -184,12 +184,6 @@ object ResourceF { ResourceMetadata(r).asJsonObject } - implicit def resourceFEncoder[A: Encoder](implicit base: BaseUri): Encoder[ResourceF[A]] = Encoder.instance { r => - ResourceIdAndTypes(r.resolvedId, r.types).asJson deepMerge - r.value.asJson deepMerge - ResourceMetadata(r).asJson - } - final private case class ResourceIdAndTypes(resolvedId: Iri, types: Set[Iri]) implicit private val idAndTypesEncoder: Encoder.AsObject[ResourceIdAndTypes] = diff --git a/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/utils/RouteFixtures.scala b/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/utils/RouteFixtures.scala index ccce575679..057979d322 100644 --- a/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/utils/RouteFixtures.scala +++ b/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/utils/RouteFixtures.scala @@ -46,8 +46,7 @@ trait RouteFixtures { contexts.supervision -> ContextValue.fromFile("contexts/supervision.json"), contexts.tags -> ContextValue.fromFile("contexts/tags.json"), contexts.version -> ContextValue.fromFile("contexts/version.json"), - contexts.quotas -> ContextValue.fromFile("contexts/quotas.json"), - contexts.bulkOperation -> ContextValue.fromFile("contexts/bulk-operation.json") + contexts.quotas -> ContextValue.fromFile("contexts/quotas.json") ) implicit val ordering: JsonKeyOrdering = diff --git a/docs/src/main/paradox/docs/delta/api/assets/files/copy-put.json b/docs/src/main/paradox/docs/delta/api/assets/files/copy-put.json deleted file mode 100644 index f4f6a287c1..0000000000 --- a/docs/src/main/paradox/docs/delta/api/assets/files/copy-put.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "@context": [ - "https://bluebrain.github.io/nexus/contexts/files.json", - "https://bluebrain.github.io/nexus/contexts/metadata.json" - ], - "@id": "http://localhost:8080/v1/resources/myorg/myproject/_/newfileid", - "@type": "File", - "_bytes": 5963969, - "_constrainedBy": "https://bluebrain.github.io/nexus/schemas/files.json", - "_createdAt": "2021-05-12T07:30:54.576Z", - "_createdBy": "http://localhost:8080/v1/anonymous", - "_deprecated": false, - "_digest": { - "_algorithm": "SHA-256", - "_value": "d14a7cb4602a2c6e1e7035809aa319d07a6d3c58303ecce7804d2e481cd4965f" - }, - "_filename": "newfile.pdf", - "_incoming": "http://localhost:8080/v1/files/myorg/myproject/newfileid/incoming", - "_location": "file:///tmp/test/nexus/myorg/myproject/c/b/5/c/4/d/8/e/newfile.pdf", - "_mediaType": "application/pdf", - "_origin": "Client", - "_outgoing": "http://localhost:8080/v1/files/myorg/myproject/newfileid/outgoing", - "_project": "http://localhost:8080/v1/projects/myorg/myproject", - "_rev": 1, - "_self": "http://localhost:8080/v1/files/myorg/myproject/newfileid", - "_storage": { - "@id": "http://localhost:8080/v1/resources/myorg/myproject/_/remote", - "@type": "RemoteDiskStorage", - "_rev": 1 - }, - "_updatedAt": "2021-05-12T07:30:54.576Z", - "_updatedBy": "http://localhost:8080/v1/anonymous", - "_uuid": "cb5c4d8e-0189-49ab-b761-c92b2d4f49d2" -} \ No newline at end of file diff --git a/docs/src/main/paradox/docs/delta/api/assets/files/copy-put.sh b/docs/src/main/paradox/docs/delta/api/assets/files/copy-put.sh deleted file mode 100644 index 1de7f91d50..0000000000 --- a/docs/src/main/paradox/docs/delta/api/assets/files/copy-put.sh +++ /dev/null @@ -1,10 +0,0 @@ -curl -X PUT \ - -H "Content-Type: application/json" \ - "http://localhost:8080/v1/files/myorg/myproject/newfileid?storage=remote" -d \ - '{ - "destinationFilename": "newfile.pdf", - "sourceProjectRef": "otherorg/otherproj", - "sourceFileId": "oldfileid", - "sourceTag": "mytag", - "sourceRev": null - }' \ No newline at end of file diff --git a/docs/src/main/paradox/docs/delta/api/files-api.md b/docs/src/main/paradox/docs/delta/api/files-api.md index 450877ac2e..b5fe65cbe5 100644 --- a/docs/src/main/paradox/docs/delta/api/files-api.md +++ b/docs/src/main/paradox/docs/delta/api/files-api.md @@ -89,62 +89,6 @@ Request Response : @@snip [created-put.json](assets/files/created-put.json) -## Create copy using POST or PUT - -Create a file copy based on a source file potentially in a different organization. No `MIME` details are necessary since this is not a file upload. Metadata such as the size and digest of the source file are preserved. - -The caller must have the following permissions: -- `files/read` on the source project. -- `storages/write` on the storage in the destination project. - -Either `POST` or `PUT` can be used to copy a file, as with other creation operations. These REST resources are in the context of the **destination** file; the one being created. -- `POST` will generate a new UUID for the file: - ``` - POST /v1/files/{org_label}/{project_label}?storage={storageId}&tag={tagName} - ``` -- `PUT` accepts a `{file_id}` from the user: - ``` - PUT /v1/files/{org_label}/{project_label}/{file_id}?storage={storageId}&tag={tagName} - ``` - -... where -- `{storageId}` optionally selects a specific storage backend for the new file. The `@type` of this storage must be `DiskStorage` or `RemoteDiskStorage`. - If omitted, the default storage of the project is used. The request will be rejected if there's not enough space on the storage. -- `{tagName}` an optional label given to the new file on its first revision. - -Both requests accept the following JSON payload: -```json -{ - "destinationFilename": "{destinationFilename}", - "sourceProjectRef": "{sourceOrg}/{sourceProj}", - "sourceFileId": "{sourceFileId}", - "sourceTag": "{sourceTagName}", - "sourceRev": "{sourceRev}" -} -``` - -... where -- `{destinationFilename}` the optional filename for the new file. If omitted, the source filename will be used. -- `{sourceOrg}` the organization label of the source file. -- `{sourceProj}` the project label of the source file. -- `{sourceFileId}` the unique identifier of the source file. -- `{sourceTagName}` the optional source revision to be fetched. -- `{sourceRev}` the optional source tag to be fetched. - -Notes: - -- The storage type of `sourceFileId` must match that of the destination file. For example, if the destination `storageId` is omitted, the source storage must be of type `DiskStorage` (the default storage type). -- `sourceTagName` and `sourceRev` cannot be simultaneously present. If neither are present, the latest revision of the source file will be used. - -**Example** - -Request -: @@snip [copy-put.sh](assets/files/copy-put.sh) - -Response -: @@snip [copy-put.json](assets/files/copy-put.json) - - ## Link using POST Brings a file existing in a storage to Nexus Delta as a file resource. This operation is supported for files using `S3Storage` and `RemoteDiskStorage`. diff --git a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/StorageError.scala b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/StorageError.scala index 1243232e15..471f50bef9 100644 --- a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/StorageError.scala +++ b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/StorageError.scala @@ -2,11 +2,9 @@ package ch.epfl.bluebrain.nexus.storage import akka.http.scaladsl.model.StatusCodes import akka.http.scaladsl.model.Uri.Path -import ch.epfl.bluebrain.nexus.delta.kernel.utils.CopyBetween import ch.epfl.bluebrain.nexus.storage.routes.StatusFrom import io.circe.generic.extras.Configuration import io.circe.generic.extras.semiauto.deriveConfiguredEncoder -import io.circe.generic.semiauto.deriveEncoder import io.circe.{Encoder, Json} import scala.annotation.nowarn @@ -97,13 +95,6 @@ object StorageError { */ final case class OperationTimedOut(override val msg: String) extends StorageError(msg) - implicit val enc: Encoder[CopyBetween] = deriveEncoder - - final case class CopyOperationFailed(failingCopy: CopyBetween) - extends StorageError( - s"Copy operation failed from source ${failingCopy.source} to destination ${failingCopy.destination}." - ) - @nowarn("cat=unused") implicit private val config: Configuration = Configuration.default.withDiscriminator("@type") diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/HttpClient.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/HttpClient.scala index dc5b3147dc..fa60adb097 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/HttpClient.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/HttpClient.scala @@ -73,7 +73,7 @@ class HttpClient private (baseUrl: Uri, httpExt: HttpExt)(implicit /** Put with no body */ def putEmptyBody[A](url: String, identity: Identity, extraHeaders: Seq[HttpHeader] = jsonHeaders)( - assertResponse: (A, HttpResponse) => Assertion + assertResponse: (A, HttpResponse) => Assertion )(implicit um: FromEntityUnmarshaller[A]): IO[Assertion] = requestAssert(PUT, url, None, identity, extraHeaders)(assertResponse) diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/StorageSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/StorageSpec.scala index ff6d22b12e..0c5cf7e6c0 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/StorageSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/StorageSpec.scala @@ -422,11 +422,11 @@ abstract class StorageSpec extends BaseIntegrationSpec { uploadFileToProjectStorage(fileInput, projectRef, storageId, rev) def uploadFileToProjectStorage( - fileInput: Input, - projRef: String, - storage: String, - rev: Option[Int] - ): ((Json, HttpResponse) => Assertion) => IO[Assertion] = { + fileInput: Input, + projRef: String, + storage: String, + rev: Option[Int] + ): ((Json, HttpResponse) => Assertion) => IO[Assertion] = { val revString = rev.map(r => s"&rev=$r").getOrElse("") deltaClient.uploadFile[Json]( s"/files/$projRef/${fileInput.fileId}?storage=nxv:$storage$revString", From e0e5b2aac37cb78a5502cfe921ee2cb47b63af47 Mon Sep 17 00:00:00 2001 From: dantb Date: Thu, 14 Dec 2023 09:52:31 +0100 Subject: [PATCH 17/18] Fix test --- .../nexus/delta/plugins/storage/files/FileFixtures.scala | 4 ++-- .../delta/plugins/storage/files/generators/FileGen.scala | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FileFixtures.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FileFixtures.scala index 57ab705e79..b66d2c4e96 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FileFixtures.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FileFixtures.scala @@ -37,7 +37,7 @@ trait FileFixtures { val generatedId2 = project.base.iri / uuid2.toString val content = "file content" - val path = FileGen.path + val path = FileGen.mkTempDir("files") def withUUIDF[T](id: UUID)(test: => T): T = (for { old <- ref.getAndSet(id) @@ -50,7 +50,7 @@ trait FileFixtures { size: Long = 12, id: UUID = uuid, projRef: ProjectRef = projectRef - ): FileAttributes = FileGen.attributes(filename, size, id, projRef) + ): FileAttributes = FileGen.attributes(filename, size, id, projRef, path) def entity(filename: String = "file.txt"): MessageEntity = Multipart diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/generators/FileGen.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/generators/FileGen.scala index 41e0a7c9b9..35ff172471 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/generators/FileGen.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/generators/FileGen.scala @@ -149,7 +149,8 @@ object FileGen { ): FileResource = state(id, project, storage, attributes, storageType, rev, deprecated, tags, createdBy, updatedBy).toResource - lazy val path = AbsolutePath(JavaFiles.createTempDirectory("files")).fold(e => throw new Exception(e), identity) + def mkTempDir(prefix: String) = + AbsolutePath(JavaFiles.createTempDirectory(prefix)).fold(e => throw new Exception(e), identity) private val digest = ComputedDigest(DigestAlgorithm.default, "e0ac3601005dfa1864f5392aabaf7d898b1b5bab854f1acb4491bcd806b76b0c") @@ -158,7 +159,8 @@ object FileGen { filename: String, size: Long, id: UUID, - projRef: ProjectRef + projRef: ProjectRef, + path: AbsolutePath ): FileAttributes = { val uuidPathSegment = id.toString.take(8).mkString("/") FileAttributes( From a8fe061f6774f0ff60fbfdd931f1b5e39f2bb649 Mon Sep 17 00:00:00 2001 From: dantb Date: Mon, 18 Dec 2023 11:38:47 +0100 Subject: [PATCH 18/18] From comments --- .../utils/TransactionalFileCopierSuite.scala | 16 ++-- .../plugins/storage/StoragePluginModule.scala | 1 - .../storage/files/batch/BatchFiles.scala | 4 +- .../files/routes/BatchFilesRoutes.scala | 15 ++-- .../operations/StorageFileRejection.scala | 3 - .../storage/files/batch/BatchFilesSuite.scala | 83 +++++++++++-------- 6 files changed, 62 insertions(+), 60 deletions(-) diff --git a/delta/kernel/src/test/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/TransactionalFileCopierSuite.scala b/delta/kernel/src/test/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/TransactionalFileCopierSuite.scala index 3625ed5bc9..587acbc9af 100644 --- a/delta/kernel/src/test/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/TransactionalFileCopierSuite.scala +++ b/delta/kernel/src/test/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/TransactionalFileCopierSuite.scala @@ -57,14 +57,14 @@ class TransactionalFileCopierSuite extends CatsEffectSuite { test("rollback by deleting file copies and directories if error thrown during a copy") { for { - (source, _) <- givenAFileExists - (failingDest, _) <- givenAFileExists - (dest1, dest3) = (genFilePath, genFilePath) - failingCopy = CopyBetween(source, failingDest) - files = NonEmptyList.of(CopyBetween(source, dest1), failingCopy, CopyBetween(source, dest3)) - error <- copier.copyAll(files).intercept[CopyOperationFailed] - _ <- List(dest1, dest3, parent(dest1), parent(dest3)).traverse(fileShouldNotExist) - _ <- fileShouldExist(failingDest) + (source, _) <- givenAFileExists + (existingFilePath, _) <- givenAFileExists + (dest1, dest3) = (genFilePath, genFilePath) + failingCopy = CopyBetween(source, existingFilePath) + files = NonEmptyList.of(CopyBetween(source, dest1), failingCopy, CopyBetween(source, dest3)) + error <- copier.copyAll(files).intercept[CopyOperationFailed] + _ <- List(dest1, dest3, parent(dest1), parent(dest3)).traverse(fileShouldNotExist) + _ <- fileShouldExist(existingFilePath) } yield assertEquals(error.failingCopy, failingCopy) } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala index ba7a17af26..670a3b991d 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala @@ -151,7 +151,6 @@ class StoragePluginModule(priority: Int) extends ModuleDef { many[ResourceShift[_, _, _]].ref[Storage.Shift] - // TODO refactor Files to depend on this rather than constructing it make[FilesLog].from { (cfg: StoragePluginConfig, xas: Transactors, clock: Clock[IO]) => ScopedEventLog(Files.definition(clock), cfg.files.eventLog, xas) } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFiles.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFiles.scala index d3098dbc82..2d3d8f0fe5 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFiles.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFiles.scala @@ -49,11 +49,11 @@ object BatchFiles { destFilesAttributes <- batchCopy.copyFiles(source, destStorage).adaptError { case e: CopyFileRejection => CopyRejection(source.project, dest.project, destStorage.id, e) } - fileResources <- evalCreateCommands(pc, dest, destStorageRef, destStorage.tpe, destFilesAttributes) + fileResources <- createFileResources(pc, dest, destStorageRef, destStorage.tpe, destFilesAttributes) } yield fileResources }.span("copyFiles") - private def evalCreateCommands( + private def createFileResources( pc: ProjectContext, dest: CopyFileDestination, destStorageRef: ResourceRef.Revision, diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesRoutes.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesRoutes.scala index 169abeb080..a85d653199 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesRoutes.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesRoutes.scala @@ -7,7 +7,6 @@ import cats.effect.IO import cats.syntax.all._ import ch.epfl.bluebrain.nexus.delta.kernel.Logger import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch.BatchFiles -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{CopyFileDestination, File, FileRejection} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.permissions.{read => Read} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{contexts, FileResource} @@ -25,7 +24,6 @@ import ch.epfl.bluebrain.nexus.delta.sdk.identities.Identities import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller import ch.epfl.bluebrain.nexus.delta.sdk.jsonld.BulkOperationResults import ch.epfl.bluebrain.nexus.delta.sdk.model.{BaseUri, IdSegment} -import kamon.instrumentation.akka.http.TracingDirectives.operationName final class BatchFilesRoutes( identities: Identities, @@ -43,7 +41,6 @@ final class BatchFilesRoutes( private val logger = Logger[BatchFilesRoutes] - import baseUri.prefixSegment import schemeDirectives.resolveProjectRef implicit val bulkOpJsonLdEnc: JsonLdEncoder[BulkOperationResults[FileResource]] = @@ -57,12 +54,10 @@ final class BatchFilesRoutes( resolveProjectRef.apply { projectRef => (post & pathEndOrSingleSlash & parameter("storage".as[IdSegment].?) & indexingMode & tagParam) { (storage, mode, tag) => - operationName(s"$prefixSegment/files/{org}/{project}") { - // Bulk create files by copying from another project - entity(as[CopyFileSource]) { c: CopyFileSource => - val copyTo = CopyFileDestination(projectRef, storage, tag) - emit(Created, copyFile(mode, c, copyTo)) - } + // Bulk create files by copying from another project + entity(as[CopyFileSource]) { c: CopyFileSource => + val copyTo = CopyFileDestination(projectRef, storage, tag) + emit(Created, copyFiles(mode, c, copyTo)) } } } @@ -71,7 +66,7 @@ final class BatchFilesRoutes( } } - private def copyFile(mode: IndexingMode, source: CopyFileSource, dest: CopyFileDestination)(implicit + private def copyFiles(mode: IndexingMode, source: CopyFileSource, dest: CopyFileDestination)(implicit caller: Caller ): IO[Either[FileRejection, BulkOperationResults[FileResource]]] = (for { diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/StorageFileRejection.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/StorageFileRejection.scala index d22c67bcbb..8e7036481a 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/StorageFileRejection.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/StorageFileRejection.scala @@ -69,9 +69,6 @@ object StorageFileRejection { extends FetchAttributeRejection(rejection.loggedDetails) } - /** - * Rejection returned when a storage cannot fetch a file's attributes - */ sealed abstract class CopyFileRejection(loggedDetails: String) extends StorageFileRejection(loggedDetails) object CopyFileRejection { diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFilesSuite.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFilesSuite.scala index 4343fec92a..30d1ee6b82 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFilesSuite.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFilesSuite.scala @@ -1,5 +1,6 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch +import cats.data.NonEmptyList import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch.BatchFilesSuite._ @@ -7,7 +8,7 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.generators.FileGen import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.mocks.BatchCopyMock import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileCommand.CreateFile import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection.CopyRejection -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileCommand, FileRejection} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes, FileCommand, FileRejection} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.CopyFileSource import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{FetchFileStorage, FileFixtures, FileResource} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StorageFixtures @@ -26,56 +27,46 @@ import scala.collection.mutable.ListBuffer class BatchFilesSuite extends NexusSuite with StorageFixtures with Generators with FileFixtures with FileGen { + private val destProj: Project = genProject() + private val (destStorageRef, destStorage) = (genRevision(), genStorage(destProj.ref, diskVal)) + private val destFileUUId = UUID.randomUUID() // Not testing UUID generation, same for all of them + private val destination = genCopyFileDestination(destProj.ref, destStorage.storage) + test("batch copying should fetch storage, perform copy and evaluate create file commands") { - val events = ListBuffer.empty[Event] - val destProj: Project = genProject() - val (destStorageRef, destStorage) = (genRevision(), genStorage(destProj.ref, diskVal)) - val fetchFileStorage = mockFetchFileStorage(destStorageRef, destStorage.storage, events) - val stubbedDestAttributes = genAttributes() - val batchCopy = BatchCopyMock.withStubbedCopyFiles(events, stubbedDestAttributes) - val destFileUUId = UUID.randomUUID() // Not testing UUID generation, same for all of them + val events = ListBuffer.empty[Event] + val fetchFileStorage = mockFetchFileStorage(destStorageRef, destStorage.storage, events) + val stubbedDestAttributes = genAttributes() + val batchCopy = BatchCopyMock.withStubbedCopyFiles(events, stubbedDestAttributes) val batchFiles: BatchFiles = mkBatchFiles(events, destProj, destFileUUId, fetchFileStorage, batchCopy) implicit val c: Caller = Caller(genUser(), Set()) - val (source, destination) = (genCopyFileSource(), genCopyFileDestination(destProj.ref, destStorage.storage)) - val obtained = batchFiles.copyFiles(source, destination).accepted - - val expectedFileIri = destProj.base.iri / destFileUUId.toString - val expectedCmds = stubbedDestAttributes.map( - CreateFile(expectedFileIri, destProj.ref, destStorageRef, destStorage.value.tpe, _, c.subject, destination.tag) - ) + val source = genCopyFileSource() - // resources returned are based on file command evaluation - assertEquals(obtained, expectedCmds.map(genFileResourceFromCmd)) + batchFiles.copyFiles(source, destination).map { obtained => + val expectedCommands = createCommandsFromFileAttributes(stubbedDestAttributes) + val expectedResources = expectedCommands.map(genFileResourceFromCmd) + val expectedCommandCalls = expectedCommands.toList.map(FileCommandEvaluated) + val expectedEvents = activeStorageFetchedAndBatchCopyCalled(source) ++ expectedCommandCalls - val expectedActiveStorageFetched = ActiveStorageFetched(destination.storage, destProj.ref, destProj.context, c) - val expectedBatchCopyCalled = BatchCopyCalled(source, destStorage.storage, c) - val expectedCommandsEvaluated = expectedCmds.toList.map(FileCommandEvaluated) - val expectedEvents = List(expectedActiveStorageFetched, expectedBatchCopyCalled) ++ expectedCommandsEvaluated - assertEquals(events.toList, expectedEvents) + assertEquals(obtained, expectedResources) + assertEquals(events.toList, expectedEvents) + } } test("copy rejections should be mapped to a file rejection") { - val events = ListBuffer.empty[Event] - val destProj: Project = genProject() - val (destStorageRef, destStorage) = (genRevision(), genStorage(destProj.ref, diskVal)) - val fetchFileStorage = mockFetchFileStorage(destStorageRef, destStorage.storage, events) - val error = TotalCopySizeTooLarge(1L, 2L, genIri()) - val batchCopy = BatchCopyMock.withError(error, events) + val events = ListBuffer.empty[Event] + val fetchFileStorage = mockFetchFileStorage(destStorageRef, destStorage.storage, events) + val error = TotalCopySizeTooLarge(1L, 2L, genIri()) + val batchCopy = BatchCopyMock.withError(error, events) val batchFiles: BatchFiles = mkBatchFiles(events, destProj, UUID.randomUUID(), fetchFileStorage, batchCopy) implicit val c: Caller = Caller(genUser(), Set()) - val sourceProj = genProject() - val (source, destination) = - (genCopyFileSource(sourceProj.ref), genCopyFileDestination(destProj.ref, destStorage.storage)) - val expectedError = CopyRejection(sourceProj.ref, destProj.ref, destStorage.id, error) + val source = genCopyFileSource() + val expectedError = CopyRejection(source.project, destProj.ref, destStorage.id, error) batchFiles.copyFiles(source, destination).interceptEquals(expectedError).accepted - val expectedActiveStorageFetched = ActiveStorageFetched(destination.storage, destProj.ref, destProj.context, c) - val expectedBatchCopyCalled = BatchCopyCalled(source, destStorage.storage, c) - val expectedEvents = List(expectedActiveStorageFetched, expectedBatchCopyCalled) - assertEquals(events.toList, expectedEvents) + assertEquals(events.toList, activeStorageFetchedAndBatchCopyCalled(source)) } def mockFetchFileStorage( @@ -103,6 +94,26 @@ class BatchFilesSuite extends NexusSuite with StorageFixtures with Generators wi FetchContextDummy(Map(proj.ref -> proj.context)).mapRejection(FileRejection.ProjectContextRejection) BatchFiles.mk(fetchFileStorage, fetchContext, evalFileCmd, batchCopy) } + + def activeStorageFetchedAndBatchCopyCalled(source: CopyFileSource)(implicit c: Caller): List[Event] = { + val expectedActiveStorageFetched = ActiveStorageFetched(destination.storage, destProj.ref, destProj.context, c) + val expectedBatchCopyCalled = BatchCopyCalled(source, destStorage.storage, c) + List(expectedActiveStorageFetched, expectedBatchCopyCalled) + } + + def createCommandsFromFileAttributes(stubbedDestAttributes: NonEmptyList[FileAttributes])(implicit + c: Caller + ): NonEmptyList[CreateFile] = stubbedDestAttributes.map( + CreateFile( + destProj.base.iri / destFileUUId.toString, + destProj.ref, + destStorageRef, + destStorage.value.tpe, + _, + c.subject, + destination.tag + ) + ) } object BatchFilesSuite {