diff --git a/build.sbt b/build.sbt index 7d99c94016..8660a3dd04 100755 --- a/build.sbt +++ b/build.sbt @@ -223,6 +223,7 @@ lazy val kernel = project scalaTest % Test ), addCompilerPlugin(kindProjector), + addCompilerPlugin(betterMonadicFor), coverageFailOnMinimum := false ) diff --git a/delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/wiring/DeltaModule.scala b/delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/wiring/DeltaModule.scala index 3aacd84032..670d494bfc 100644 --- a/delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/wiring/DeltaModule.scala +++ b/delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/wiring/DeltaModule.scala @@ -79,6 +79,7 @@ class DeltaModule(appCfg: AppConfig, config: Config)(implicit classLoader: Class make[RemoteContextResolution].named("aggregate").fromEffect { (otherCtxResolutions: Set[RemoteContextResolution]) => for { + bulkOpCtx <- ContextValue.fromFile("contexts/bulk-operation.json") errorCtx <- ContextValue.fromFile("contexts/error.json") metadataCtx <- ContextValue.fromFile("contexts/metadata.json") searchCtx <- ContextValue.fromFile("contexts/search.json") @@ -96,7 +97,8 @@ class DeltaModule(appCfg: AppConfig, config: Config)(implicit classLoader: Class contexts.remoteContexts -> remoteContextsCtx, contexts.tags -> tagsCtx, contexts.version -> versionCtx, - contexts.validation -> validationCtx + contexts.validation -> validationCtx, + contexts.bulkOperation -> bulkOpCtx ) .merge(otherCtxResolutions.toSeq: _*) } diff --git a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/files/CopyFiles.scala b/delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/TransactionalFileCopier.scala similarity index 52% rename from storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/files/CopyFiles.scala rename to delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/TransactionalFileCopier.scala index ed6eddb780..254d5b5273 100644 --- a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/files/CopyFiles.scala +++ b/delta/kernel/src/main/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/TransactionalFileCopier.scala @@ -1,27 +1,41 @@ -package ch.epfl.bluebrain.nexus.storage.files +package ch.epfl.bluebrain.nexus.delta.kernel.utils import cats.data.NonEmptyList import cats.effect.{IO, Ref} import cats.implicits._ -import ch.epfl.bluebrain.nexus.storage.StorageError.CopyOperationFailed +import ch.epfl.bluebrain.nexus.delta.kernel.Logger +import ch.epfl.bluebrain.nexus.delta.kernel.error.Rejection import fs2.io.file.{CopyFlag, CopyFlags, Files, Path} -trait CopyFiles { - def copyValidated(files: NonEmptyList[ValidatedCopyFile]): IO[Unit] +trait TransactionalFileCopier { + def copyAll(files: NonEmptyList[CopyBetween]): IO[Unit] } -object CopyFiles { - def mk(): CopyFiles = files => - copyAll(files.map(v => CopyBetween(Path.fromNioPath(v.absSourcePath), Path.fromNioPath(v.absDestPath)))) +final case class CopyBetween(source: Path, destination: Path) - def copyAll(files: NonEmptyList[CopyBetween]): IO[Unit] = +final case class CopyOperationFailed(failingCopy: CopyBetween, e: Throwable) extends Rejection { + override def reason: String = + s"Copy operation failed from source ${failingCopy.source} to destination ${failingCopy.destination}. Underlying error: $e" +} + +object TransactionalFileCopier { + + private val logger = Logger[TransactionalFileCopier] + + def mk(): TransactionalFileCopier = files => copyAll(files) + + private def copyAll(files: NonEmptyList[CopyBetween]): IO[Unit] = Ref.of[IO, Option[CopyOperationFailed]](None).flatMap { errorRef => files .parTraverse { case c @ CopyBetween(source, dest) => - copySingle(source, dest).onError(_ => errorRef.set(Some(CopyOperationFailed(c)))) + copySingle(source, dest).onError(e => errorRef.set(Some(CopyOperationFailed(c, e)))) } .void - .handleErrorWith(_ => rollbackCopiesAndRethrow(errorRef, files.map(_.destination))) + .handleErrorWith { e => + val destinations = files.map(_.destination) + logger.error(e)(s"Transactional files copy failed, deleting created files: ${destinations}") >> + rollbackCopiesAndRethrow(errorRef, destinations) + } } def parent(p: Path): Path = Path.fromNioPath(p.toNioPath.getParent) diff --git a/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/files/CopyFileSuite.scala b/delta/kernel/src/test/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/TransactionalFileCopierSuite.scala similarity index 81% rename from storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/files/CopyFileSuite.scala rename to delta/kernel/src/test/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/TransactionalFileCopierSuite.scala index d962ea3709..587acbc9af 100644 --- a/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/files/CopyFileSuite.scala +++ b/delta/kernel/src/test/scala/ch/epfl/bluebrain/nexus/delta/kernel/utils/TransactionalFileCopierSuite.scala @@ -1,10 +1,9 @@ -package ch.epfl.bluebrain.nexus.storage.files +package ch.epfl.bluebrain.nexus.delta.kernel.utils import cats.data.NonEmptyList import cats.effect.IO import cats.syntax.all._ -import ch.epfl.bluebrain.nexus.storage.StorageError.CopyOperationFailed -import ch.epfl.bluebrain.nexus.storage.files.CopyFiles.parent +import ch.epfl.bluebrain.nexus.delta.kernel.utils.TransactionalFileCopier.parent import fs2.io.file.PosixPermission._ import fs2.io.file._ import munit.CatsEffectSuite @@ -12,19 +11,21 @@ import munit.catseffect.IOFixture import java.util.UUID -class CopyFileSuite extends CatsEffectSuite { +class TransactionalFileCopierSuite extends CatsEffectSuite { val myFixture: IOFixture[Path] = ResourceSuiteLocalFixture("create-temp-dir-fixture", Files[IO].tempDirectory) override def munitFixtures: List[IOFixture[Path]] = List(myFixture) lazy val tempDir: Path = myFixture() + val copier: TransactionalFileCopier = TransactionalFileCopier.mk() + test("successfully copy contents of multiple files") { for { (source1, source1Contents) <- givenAFileExists (source2, source2Contents) <- givenAFileExists (dest1, dest2, dest3) = (genFilePath, genFilePath, genFilePath) files = NonEmptyList.of(CopyBetween(source1, dest1), CopyBetween(source2, dest2), CopyBetween(source1, dest3)) - _ <- CopyFiles.copyAll(files) + _ <- copier.copyAll(files) _ <- fileShouldExistWithContents(source1Contents, dest1) _ <- fileShouldExistWithContents(source1Contents, dest3) _ <- fileShouldExistWithContents(source2Contents, dest2) @@ -37,7 +38,7 @@ class CopyFileSuite extends CatsEffectSuite { sourceAttr <- Files[IO].getBasicFileAttributes(source) dest = genFilePath files = NonEmptyList.of(CopyBetween(source, dest)) - _ <- CopyFiles.copyAll(files) + _ <- copier.copyAll(files) _ <- fileShouldExistWithContentsAndAttributes(dest, contents, sourceAttr) } yield () } @@ -49,22 +50,22 @@ class CopyFileSuite extends CatsEffectSuite { (source, _) <- givenAFileWithPermissions(sourcePermissions) dest = genFilePath files = NonEmptyList.of(CopyBetween(source, dest)) - _ <- CopyFiles.copyAll(files) + _ <- copier.copyAll(files) _ <- fileShouldExistWithPermissions(dest, sourcePermissions) } yield () } test("rollback by deleting file copies and directories if error thrown during a copy") { for { - (source, _) <- givenAFileExists - (failingDest, _) <- givenAFileExists - (dest1, dest3) = (genFilePath, genFilePath) - failingCopy = CopyBetween(source, failingDest) - files = NonEmptyList.of(CopyBetween(source, dest1), failingCopy, CopyBetween(source, dest3)) - error <- CopyFiles.copyAll(files).intercept[CopyOperationFailed] - _ <- List(dest1, dest3, parent(dest1), parent(dest3)).traverse(fileShouldNotExist) - _ <- fileShouldExist(failingDest) - } yield assertEquals(error, CopyOperationFailed(failingCopy)) + (source, _) <- givenAFileExists + (existingFilePath, _) <- givenAFileExists + (dest1, dest3) = (genFilePath, genFilePath) + failingCopy = CopyBetween(source, existingFilePath) + files = NonEmptyList.of(CopyBetween(source, dest1), failingCopy, CopyBetween(source, dest3)) + error <- copier.copyAll(files).intercept[CopyOperationFailed] + _ <- List(dest1, dest3, parent(dest1), parent(dest3)).traverse(fileShouldNotExist) + _ <- fileShouldExist(existingFilePath) + } yield assertEquals(error.failingCopy, failingCopy) } test("rollback read-only files upon failure") { @@ -76,10 +77,10 @@ class CopyFileSuite extends CatsEffectSuite { dest2 = genFilePath failingCopy = CopyBetween(source, failingDest) files = NonEmptyList.of(CopyBetween(source, dest2), failingCopy) - error <- CopyFiles.copyAll(files).intercept[CopyOperationFailed] + error <- copier.copyAll(files).intercept[CopyOperationFailed] _ <- List(dest2, parent(dest2)).traverse(fileShouldNotExist) _ <- fileShouldExist(failingDest) - } yield assertEquals(error, CopyOperationFailed(failingCopy)) + } yield assertEquals(error.failingCopy, failingCopy) } def genFilePath: Path = tempDir / genString / s"$genString.txt" diff --git a/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveDownloadSpec.scala b/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveDownloadSpec.scala index 567270e4c7..c9ffed6f23 100644 --- a/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveDownloadSpec.scala +++ b/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveDownloadSpec.scala @@ -14,10 +14,11 @@ import ch.epfl.bluebrain.nexus.delta.plugins.archive.model.ArchiveReference.{Fil import ch.epfl.bluebrain.nexus.delta.plugins.archive.model.ArchiveRejection.{InvalidFileSelf, ResourceNotFound} import ch.epfl.bluebrain.nexus.delta.plugins.archive.model.{ArchiveRejection, ArchiveValue} import ch.epfl.bluebrain.nexus.delta.plugins.storage.RemoteContextResolutionFixture +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.generators.FileGen import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes.FileAttributesOrigin.Client import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection.FileNotFound import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{Digest, FileAttributes} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{schemas, FileGen} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.schemas import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StorageFixtures import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.AbsolutePath import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri diff --git a/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveRoutesSpec.scala b/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveRoutesSpec.scala index 6398723743..91a3ab3e83 100644 --- a/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveRoutesSpec.scala +++ b/delta/plugins/archive/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/archive/ArchiveRoutesSpec.scala @@ -14,12 +14,13 @@ import ch.epfl.bluebrain.nexus.delta.kernel.utils.{StatefulUUIDF, UUIDF} import ch.epfl.bluebrain.nexus.delta.plugins.archive.FileSelf.ParsingError.InvalidPath import ch.epfl.bluebrain.nexus.delta.plugins.archive.model.ArchiveRejection.ProjectContextRejection import ch.epfl.bluebrain.nexus.delta.plugins.archive.routes.ArchiveRoutes +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.generators.FileGen import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest.ComputedDigest import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes.FileAttributesOrigin.Client import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection.FileNotFound import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{File, FileAttributes} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.FilesRoutesSpec -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{schemas, FileGen} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.schemas import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StorageFixtures import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.DigestAlgorithm import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala index 19efc3c516..670a3b991d 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/StoragePluginModule.scala @@ -3,18 +3,22 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage import akka.actor import akka.actor.typed.ActorSystem import cats.effect.{Clock, IO} -import ch.epfl.bluebrain.nexus.delta.kernel.utils.{ClasspathResourceLoader, UUIDF} +import ch.epfl.bluebrain.nexus.delta.kernel.utils.{ClasspathResourceLoader, TransactionalFileCopier, UUIDF} import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.client.ElasticSearchClient import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.config.ElasticSearchViewsConfig import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.Files +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.Files.FilesLog +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch.{BatchCopy, BatchFiles} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.contexts.{files => fileCtxId} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model._ -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.FilesRoutes +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.{BatchFilesRoutes, FilesRoutes} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.schemas.{files => filesSchemaId} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StoragesConfig.StorageTypeConfig import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.contexts.{storages => storageCtxId, storagesMetadata => storageMetaCtxId} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageAccess +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.DiskStorageCopyFiles +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.RemoteDiskStorageCopyFiles import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.RemoteDiskStorageClient import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.routes.StoragesRoutes import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.schemas.{storage => storagesSchemaId} @@ -40,7 +44,7 @@ import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContext.ContextRejection import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.ApiMappings import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverContextResolution import ch.epfl.bluebrain.nexus.delta.sdk.sse.SseEncoder -import ch.epfl.bluebrain.nexus.delta.sourcing.Transactors +import ch.epfl.bluebrain.nexus.delta.sourcing.{ScopedEventLog, Transactors} import ch.epfl.bluebrain.nexus.delta.sourcing.model.Label import ch.epfl.bluebrain.nexus.delta.sourcing.stream.Supervisor import com.typesafe.config.Config @@ -147,6 +151,10 @@ class StoragePluginModule(priority: Int) extends ModuleDef { many[ResourceShift[_, _, _]].ref[Storage.Shift] + make[FilesLog].from { (cfg: StoragePluginConfig, xas: Transactors, clock: Clock[IO]) => + ScopedEventLog(Files.definition(clock), cfg.files.eventLog, xas) + } + make[Files] .fromEffect { ( @@ -185,6 +193,41 @@ class StoragePluginModule(priority: Int) extends ModuleDef { } } + make[TransactionalFileCopier].fromValue(TransactionalFileCopier.mk()) + + make[DiskStorageCopyFiles].from { copier: TransactionalFileCopier => DiskStorageCopyFiles.mk(copier) } + + make[RemoteDiskStorageCopyFiles].from { client: RemoteDiskStorageClient => RemoteDiskStorageCopyFiles.mk(client) } + + make[BatchCopy].from { + ( + files: Files, + storages: Storages, + aclCheck: AclCheck, + storagesStatistics: StoragesStatistics, + diskCopy: DiskStorageCopyFiles, + remoteDiskCopy: RemoteDiskStorageCopyFiles, + uuidF: UUIDF + ) => + BatchCopy.mk(files, storages, aclCheck, storagesStatistics, diskCopy, remoteDiskCopy)(uuidF) + } + + make[BatchFiles].from { + ( + fetchContext: FetchContext[ContextRejection], + files: Files, + filesLog: FilesLog, + batchCopy: BatchCopy, + uuidF: UUIDF + ) => + BatchFiles.mk( + files, + fetchContext.mapRejection(FileRejection.ProjectContextRejection), + FilesLog.eval(filesLog), + batchCopy + )(uuidF) + } + make[FilesRoutes].from { ( cfg: StoragePluginConfig, @@ -209,6 +252,28 @@ class StoragePluginModule(priority: Int) extends ModuleDef { ) } + make[BatchFilesRoutes].from { + ( + cfg: StoragePluginConfig, + identities: Identities, + aclCheck: AclCheck, + batchFiles: BatchFiles, + schemeDirectives: DeltaSchemeDirectives, + indexingAction: AggregateIndexingAction, + shift: File.Shift, + baseUri: BaseUri, + cr: RemoteContextResolution @Id("aggregate"), + ordering: JsonKeyOrdering + ) => + val storageConfig = cfg.storages.storageTypeConfig + new BatchFilesRoutes(identities, aclCheck, batchFiles, schemeDirectives, indexingAction(_, _, _)(shift))( + baseUri, + storageConfig, + cr, + ordering + ) + } + make[File.Shift].from { (files: Files, base: BaseUri, storageTypeConfig: StorageTypeConfig) => File.shift(files)(base, storageTypeConfig) } @@ -283,4 +348,8 @@ class StoragePluginModule(priority: Int) extends ModuleDef { many[PriorityRoute].add { (fileRoutes: FilesRoutes) => PriorityRoute(priority, fileRoutes.routes, requiresStrictEntity = false) } + + many[PriorityRoute].add { (batchFileRoutes: BatchFilesRoutes) => + PriorityRoute(priority, batchFileRoutes.routes, requiresStrictEntity = false) + } } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FetchFileResource.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FetchFileResource.scala new file mode 100644 index 0000000000..d77edd4c01 --- /dev/null +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FetchFileResource.scala @@ -0,0 +1,15 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files + +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileId + +trait FetchFileResource { + + /** + * Fetch the last version of a file + * + * @param id + * the identifier that will be expanded to the Iri of the file with its optional rev/tag + */ + def fetch(id: FileId): IO[FileResource] +} diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FetchFileStorage.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FetchFileStorage.scala new file mode 100644 index 0000000000..d97b030b03 --- /dev/null +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FetchFileStorage.scala @@ -0,0 +1,14 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files + +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage +import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller +import ch.epfl.bluebrain.nexus.delta.sdk.model._ +import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.ProjectContext +import ch.epfl.bluebrain.nexus.delta.sourcing.model.{ProjectRef, ResourceRef} + +trait FetchFileStorage { + def fetchAndValidateActiveStorage(storageIdOpt: Option[IdSegment], ref: ProjectRef, pc: ProjectContext)(implicit + caller: Caller + ): IO[(ResourceRef.Revision, Storage)] +} diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala index fb575da2b3..e03e7d7b7e 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/Files.scala @@ -3,7 +3,7 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.files import akka.actor.typed.ActorSystem import akka.actor.{ActorSystem => ClassicActorSystem} import akka.http.scaladsl.model.ContentTypes.`application/octet-stream` -import akka.http.scaladsl.model.{ContentType, HttpEntity, Uri} +import akka.http.scaladsl.model.{BodyPartEntity, ContentType, HttpEntity, Uri} import cats.effect.{Clock, IO} import cats.syntax.all._ import ch.epfl.bluebrain.nexus.delta.kernel.cache.LocalCache @@ -24,7 +24,7 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{DigestAlgor import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.{FetchAttributeRejection, FetchFileRejection, SaveFileRejection} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.RemoteDiskStorageClient -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{Storages, StoragesStatistics} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{FetchStorage, Storages, StoragesStatistics} import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.ContextValue import ch.epfl.bluebrain.nexus.delta.sdk.AkkaSource @@ -58,14 +58,15 @@ final class Files( log: FilesLog, aclCheck: AclCheck, fetchContext: FetchContext[FileRejection], - storages: Storages, + storages: FetchStorage, storagesStatistics: StoragesStatistics, remoteDiskStorageClient: RemoteDiskStorageClient, config: StorageTypeConfig )(implicit uuidF: UUIDF, system: ClassicActorSystem -) { +) extends FetchFileStorage + with FetchFileResource { implicit private val kamonComponent: KamonMetricComponent = KamonMetricComponent(entityType.value) @@ -97,7 +98,7 @@ final class Files( pc <- fetchContext.onCreate(projectRef) iri <- generateId(pc) _ <- test(CreateFile(iri, projectRef, testStorageRef, testStorageType, testAttributes, caller.subject, tag)) - (storageRef, storage) <- fetchActiveStorage(storageId, projectRef, pc) + (storageRef, storage) <- fetchAndValidateActiveStorage(storageId, projectRef, pc) attributes <- extractFileAttributes(iri, entity, storage) res <- eval(CreateFile(iri, projectRef, storageRef, storage.tpe, attributes, caller.subject, tag)) } yield res @@ -126,7 +127,7 @@ final class Files( for { (iri, pc) <- id.expandIri(fetchContext.onCreate) _ <- test(CreateFile(iri, id.project, testStorageRef, testStorageType, testAttributes, caller.subject, tag)) - (storageRef, storage) <- fetchActiveStorage(storageId, id.project, pc) + (storageRef, storage) <- fetchAndValidateActiveStorage(storageId, id.project, pc) attributes <- extractFileAttributes(iri, entity, storage) res <- eval(CreateFile(iri, id.project, storageRef, storage.tpe, attributes, caller.subject, tag)) } yield res @@ -219,7 +220,7 @@ final class Files( for { (iri, pc) <- id.expandIri(fetchContext.onModify) _ <- test(UpdateFile(iri, id.project, testStorageRef, testStorageType, testAttributes, rev, caller.subject, tag)) - (storageRef, storage) <- fetchActiveStorage(storageId, id.project, pc) + (storageRef, storage) <- fetchAndValidateActiveStorage(storageId, id.project, pc) attributes <- extractFileAttributes(iri, entity, storage) res <- eval(UpdateFile(iri, id.project, storageRef, storage.tpe, attributes, rev, caller.subject, tag)) } yield res @@ -255,7 +256,7 @@ final class Files( for { (iri, pc) <- id.expandIri(fetchContext.onModify) _ <- test(UpdateFile(iri, id.project, testStorageRef, testStorageType, testAttributes, rev, caller.subject, tag)) - (storageRef, storage) <- fetchActiveStorage(storageId, id.project, pc) + (storageRef, storage) <- fetchAndValidateActiveStorage(storageId, id.project, pc) resolvedFilename <- IO.fromOption(filename.orElse(path.lastSegment))(InvalidFileLink(iri)) description <- FileDescription(resolvedFilename, mediaType) attributes <- linkFile(storage, path, description, iri) @@ -378,20 +379,11 @@ final class Files( FetchRejection(fileId, storage.id, e) } - /** - * Fetch the last version of a file - * - * @param id - * the identifier that will be expanded to the Iri of the file with its optional rev/tag - * @param project - * the project where the storage belongs - */ - def fetch(id: FileId): IO[FileResource] = { - for { + override def fetch(id: FileId): IO[FileResource] = + (for { (iri, _) <- id.expandIri(fetchContext.onRead) state <- fetchState(id, iri) - } yield state.toResource - }.span("fetchFile") + } yield state.toResource).span("fetchFile") private def fetchState(id: FileId, iri: Iri): IO[FileState] = { val notFound = FileNotFound(iri, id.project) @@ -414,7 +406,7 @@ final class Files( )(implicit caller: Caller): IO[FileResource] = for { _ <- test(CreateFile(iri, ref, testStorageRef, testStorageType, testAttributes, caller.subject, tag)) - (storageRef, storage) <- fetchActiveStorage(storageId, ref, pc) + (storageRef, storage) <- fetchAndValidateActiveStorage(storageId, ref, pc) resolvedFilename <- IO.fromOption(filename.orElse(path.lastSegment))(InvalidFileLink(iri)) description <- FileDescription(resolvedFilename, mediaType) attributes <- linkFile(storage, path, description, iri) @@ -426,13 +418,12 @@ final class Files( .apply(path, desc) .adaptError { case e: StorageFileRejection => LinkRejection(fileId, storage.id, e) } - private def eval(cmd: FileCommand): IO[FileResource] = - log.evaluate(cmd.project, cmd.id, cmd).map(_._2.toResource) + private def eval(cmd: FileCommand): IO[FileResource] = FilesLog.eval(log)(cmd) private def test(cmd: FileCommand) = log.dryRun(cmd.project, cmd.id, cmd) - private def fetchActiveStorage(storageIdOpt: Option[IdSegment], ref: ProjectRef, pc: ProjectContext)(implicit - caller: Caller + override def fetchAndValidateActiveStorage(storageIdOpt: Option[IdSegment], ref: ProjectRef, pc: ProjectContext)( + implicit caller: Caller ): IO[(ResourceRef.Revision, Storage)] = storageIdOpt match { case Some(storageId) => @@ -456,26 +447,27 @@ final class Files( private def extractFileAttributes(iri: Iri, entity: HttpEntity, storage: Storage): IO[FileAttributes] = for { - storageAvailableSpace <- storage.storageValue.capacity.fold(IO.none[Long]) { capacity => - storagesStatistics - .get(storage.id, storage.project) - .redeem( - _ => Some(capacity), - stat => Some(capacity - stat.spaceUsed) - ) - } - (description, source) <- formDataExtractor(iri, entity, storage.storageValue.maxFileSize, storageAvailableSpace) - attributes <- SaveFile(storage, remoteDiskStorageClient, config) - .apply(description, source) - .adaptError { case e: SaveFileRejection => SaveRejection(iri, storage.id, e) } + (description, source) <- extractFormData(iri, storage, entity) + attributes <- saveFile(iri, storage, description, source) } yield attributes - private def expandStorageIri(segment: IdSegment, pc: ProjectContext): IO[Iri] = + private def extractFormData(iri: Iri, storage: Storage, entity: HttpEntity): IO[(FileDescription, BodyPartEntity)] = + for { + storageAvailableSpace <- storagesStatistics.getStorageAvailableSpace(storage) + (description, source) <- formDataExtractor(iri, entity, storage.storageValue.maxFileSize, storageAvailableSpace) + } yield (description, source) + + private def saveFile(iri: Iri, storage: Storage, description: FileDescription, source: BodyPartEntity) = + SaveFile(storage, remoteDiskStorageClient, config) + .apply(description, source) + .adaptError { case e: SaveFileRejection => SaveRejection(iri, storage.id, e) } + + private def expandStorageIri(segment: IdSegment, pc: ProjectContext): IO[Iri] = Storages.expandIri(segment, pc).adaptError { case s: StorageRejection => WrappedStorageRejection(s) } - private def generateId(pc: ProjectContext)(implicit uuidF: UUIDF): IO[Iri] = + private def generateId(pc: ProjectContext): IO[Iri] = uuidF().map(uuid => pc.base.iri / uuid.toString) /** @@ -594,6 +586,11 @@ object Files { type FilesLog = ScopedEventLog[Iri, FileState, FileCommand, FileEvent, FileRejection] + object FilesLog { + def eval(log: FilesLog)(cmd: FileCommand): IO[FileResource] = + log.evaluate(cmd.project, cmd.id, cmd).map(_._2.toResource) + } + private[files] def next( state: Option[FileState], event: FileEvent @@ -757,7 +754,7 @@ object Files { def apply( fetchContext: FetchContext[FileRejection], aclCheck: AclCheck, - storages: Storages, + storages: FetchStorage, storagesStatistics: StoragesStatistics, xas: Transactors, storageTypeConfig: StorageTypeConfig, @@ -795,5 +792,4 @@ object Files { ) .void } - } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopy.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopy.scala new file mode 100644 index 0000000000..37c94fd763 --- /dev/null +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopy.scala @@ -0,0 +1,125 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch + +import cats.data.NonEmptyList +import cats.effect.IO +import cats.implicits.toFunctorOps +import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.FetchFileResource +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection._ +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model._ +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.CopyFileSource +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.{DiskStorage, RemoteDiskStorage} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{Storage, StorageType} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.CopyFileRejection +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.CopyFileRejection._ +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.{DiskCopyDetails, DiskStorageCopyFiles} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.RemoteDiskStorageCopyFiles +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.model.RemoteDiskCopyDetails +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{FetchStorage, StoragesStatistics} +import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri +import ch.epfl.bluebrain.nexus.delta.sdk.acls.AclCheck +import ch.epfl.bluebrain.nexus.delta.sdk.error.ServiceError.AuthorizationFailed +import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller +import shapeless.syntax.typeable.typeableOps + +trait BatchCopy { + def copyFiles(source: CopyFileSource, destStorage: Storage)(implicit + c: Caller + ): IO[NonEmptyList[FileAttributes]] +} + +object BatchCopy { + def mk( + fetchFile: FetchFileResource, + fetchStorage: FetchStorage, + aclCheck: AclCheck, + storagesStatistics: StoragesStatistics, + diskCopy: DiskStorageCopyFiles, + remoteDiskCopy: RemoteDiskStorageCopyFiles + )(implicit uuidF: UUIDF): BatchCopy = new BatchCopy { + + override def copyFiles(source: CopyFileSource, destStorage: Storage)(implicit + c: Caller + ): IO[NonEmptyList[FileAttributes]] = + destStorage match { + case disk: Storage.DiskStorage => copyToDiskStorage(source, disk) + case remote: Storage.RemoteDiskStorage => copyToRemoteStorage(source, remote) + case s3: Storage.S3Storage => unsupported(s3.tpe) + } + + private def copyToRemoteStorage(source: CopyFileSource, dest: RemoteDiskStorage)(implicit c: Caller) = + for { + remoteCopyDetails <- source.files.traverse(fetchRemoteCopyDetails(dest, _)) + _ <- validateFilesForStorage(dest, remoteCopyDetails.map(_.sourceAttributes.bytes)) + attributes <- remoteDiskCopy.copyFiles(dest, remoteCopyDetails) + } yield attributes + + private def copyToDiskStorage(source: CopyFileSource, dest: DiskStorage)(implicit c: Caller) = + for { + diskCopyDetails <- source.files.traverse(fetchDiskCopyDetails(dest, _)) + _ <- validateFilesForStorage(dest, diskCopyDetails.map(_.sourceAttributes.bytes)) + attributes <- diskCopy.copyFiles(dest, diskCopyDetails) + } yield attributes + + private def validateFilesForStorage(destStorage: Storage, sourcesBytes: NonEmptyList[Long]): IO[Unit] = { + val maxSize = destStorage.storageValue.maxFileSize + for { + _ <- IO.raiseWhen(sourcesBytes.exists(_ > maxSize))(SourceFileTooLarge(maxSize, destStorage.id)) + _ <- validateRemainingStorageCapacity(destStorage, sourcesBytes) + } yield () + } + + private def validateRemainingStorageCapacity(destStorage: Storage, sourcesBytes: NonEmptyList[Long]) = + for { + spaceLeft <- storagesStatistics.getStorageAvailableSpace(destStorage) + totalSize = sourcesBytes.toList.sum + _ <- spaceLeft + .collectFirst { case s if s < totalSize => notEnoughSpace(totalSize, s, destStorage.id) } + .getOrElse(IO.unit) + } yield () + + private def fetchDiskCopyDetails(destStorage: DiskStorage, fileId: FileId)(implicit c: Caller) = + for { + (file, sourceStorage) <- fetchFileAndValidateStorage(fileId) + destinationDesc <- FileDescription(file.attributes.filename, file.attributes.mediaType) + _ <- validateDiskStorage(destStorage, sourceStorage) + } yield DiskCopyDetails(destStorage, destinationDesc, file.attributes) + + private def validateDiskStorage(destStorage: DiskStorage, sourceStorage: Storage) = + sourceStorage + .narrowTo[DiskStorage] + .as(IO.unit) + .getOrElse(differentStorageTypeError(destStorage, sourceStorage)) + + private def fetchRemoteCopyDetails(destStorage: RemoteDiskStorage, fileId: FileId)(implicit c: Caller) = + for { + (file, sourceStorage) <- fetchFileAndValidateStorage(fileId) + destinationDesc <- FileDescription(file.attributes.filename, file.attributes.mediaType) + sourceBucket <- validateRemoteStorage(destStorage, sourceStorage) + } yield RemoteDiskCopyDetails(destStorage, destinationDesc, sourceBucket, file.attributes) + + private def validateRemoteStorage(destStorage: RemoteDiskStorage, sourceStorage: Storage) = + sourceStorage + .narrowTo[RemoteDiskStorage] + .map(remote => IO.pure(remote.value.folder)) + .getOrElse(differentStorageTypeError(destStorage, sourceStorage)) + + private def differentStorageTypeError[A](destStorage: Storage, sourceStorage: Storage) = + IO.raiseError[A](DifferentStorageTypes(sourceStorage.id, sourceStorage.tpe, destStorage.tpe)) + + private def unsupported(tpe: StorageType) = IO.raiseError(CopyFileRejection.UnsupportedOperation(tpe)) + + private def notEnoughSpace(totalSize: Long, spaceLeft: Long, destStorage: Iri) = + IO.raiseError(TotalCopySizeTooLarge(totalSize, spaceLeft, destStorage)) + + private def fetchFileAndValidateStorage(id: FileId)(implicit c: Caller) = { + for { + file <- fetchFile.fetch(id) + sourceStorage <- fetchStorage.fetch(file.value.storage, id.project) + perm = sourceStorage.value.storageValue.readPermission + _ <- aclCheck.authorizeForOr(id.project, perm)(AuthorizationFailed(id.project, perm)) + } yield (file.value, sourceStorage.value) + } + } + +} diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFiles.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFiles.scala new file mode 100644 index 0000000000..2d3d8f0fe5 --- /dev/null +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFiles.scala @@ -0,0 +1,81 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch + +import cats.data.NonEmptyList +import cats.effect.IO +import cats.implicits._ +import ch.epfl.bluebrain.nexus.delta.kernel.Logger +import ch.epfl.bluebrain.nexus.delta.kernel.kamon.KamonMetricComponent +import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.Files.entityType +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileCommand._ +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection.CopyRejection +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model._ +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.CopyFileSource +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{FetchFileStorage, FileResource} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageType +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.CopyFileRejection +import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri +import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller +import ch.epfl.bluebrain.nexus.delta.sdk.implicits._ +import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContext +import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.ProjectContext +import ch.epfl.bluebrain.nexus.delta.sourcing.model.ResourceRef + +trait BatchFiles { + def copyFiles( + source: CopyFileSource, + dest: CopyFileDestination + )(implicit c: Caller): IO[NonEmptyList[FileResource]] +} + +object BatchFiles { + def mk( + fetchFileStorage: FetchFileStorage, + fetchContext: FetchContext[FileRejection], + evalFileCommand: CreateFile => IO[FileResource], + batchCopy: BatchCopy + )(implicit uuidF: UUIDF): BatchFiles = new BatchFiles { + + private val logger = Logger[BatchFiles] + + implicit private val kamonComponent: KamonMetricComponent = KamonMetricComponent(entityType.value) + + override def copyFiles(source: CopyFileSource, dest: CopyFileDestination)(implicit + c: Caller + ): IO[NonEmptyList[FileResource]] = { + for { + pc <- fetchContext.onCreate(dest.project) + (destStorageRef, destStorage) <- fetchFileStorage.fetchAndValidateActiveStorage(dest.storage, dest.project, pc) + destFilesAttributes <- batchCopy.copyFiles(source, destStorage).adaptError { case e: CopyFileRejection => + CopyRejection(source.project, dest.project, destStorage.id, e) + } + fileResources <- createFileResources(pc, dest, destStorageRef, destStorage.tpe, destFilesAttributes) + } yield fileResources + }.span("copyFiles") + + private def createFileResources( + pc: ProjectContext, + dest: CopyFileDestination, + destStorageRef: ResourceRef.Revision, + destStorageTpe: StorageType, + destFilesAttributes: NonEmptyList[FileAttributes] + )(implicit c: Caller): IO[NonEmptyList[FileResource]] = + destFilesAttributes.traverse { destFileAttributes => + for { + iri <- generateId(pc) + command = + CreateFile(iri, dest.project, destStorageRef, destStorageTpe, destFileAttributes, c.subject, dest.tag) + resource <- evalCreateCommand(command) + } yield resource + } + + private def generateId(pc: ProjectContext): IO[Iri] = + uuidF().map(uuid => pc.base.iri / uuid.toString) + + private def evalCreateCommand(command: CreateFile) = + evalFileCommand(command).onError { e => + logger.error(e)(s"Failed storing file copy event, file must be manually deleted: $command") + } + } + +} diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/CopyFileDestination.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/CopyFileDestination.scala new file mode 100644 index 0000000000..03426b5b0b --- /dev/null +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/CopyFileDestination.scala @@ -0,0 +1,21 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model + +import ch.epfl.bluebrain.nexus.delta.sdk.model.IdSegment +import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef +import ch.epfl.bluebrain.nexus.delta.sourcing.model.Tag.UserTag + +/** + * Details for the files we're creating in the copy + * + * @param project + * Orgnization and project for the new file + * @param storage + * Optional storage for the new file which must have the same type as the source file's storage + * @param tag + * Optional tag to create the new file with + */ +final case class CopyFileDestination( + project: ProjectRef, + storage: Option[IdSegment], + tag: Option[UserTag] +) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/File.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/File.scala index e33bda375c..b4a01157a4 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/File.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/File.scala @@ -48,15 +48,16 @@ object File { final case class Metadata(tags: List[UserTag]) - implicit def fileEncoder(implicit config: StorageTypeConfig): Encoder[File] = { file => - implicit val storageType: StorageType = file.storageType - val storageJson = Json.obj( - keywords.id -> file.storage.iri.asJson, - keywords.tpe -> storageType.iri.asJson, - "_rev" -> file.storage.rev.asJson - ) - file.attributes.asJson.mapObject(_.add("_storage", storageJson)) - } + implicit def fileEncoder(implicit config: StorageTypeConfig): Encoder.AsObject[File] = + Encoder.encodeJsonObject.contramapObject { file => + implicit val storageType: StorageType = file.storageType + val storageJson = Json.obj( + keywords.id -> file.storage.iri.asJson, + keywords.tpe -> storageType.iri.asJson, + "_rev" -> file.storage.rev.asJson + ) + file.attributes.asJsonObject.add("_storage", storageJson) + } implicit def fileJsonLdEncoder(implicit config: StorageTypeConfig): JsonLdEncoder[File] = JsonLdEncoder.computeFromCirce(_.id, Files.context) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileAttributes.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileAttributes.scala index 426d9334e2..65c61683a4 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileAttributes.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileAttributes.scala @@ -28,7 +28,7 @@ import scala.annotation.nowarn * @param mediaType * the optional media type of the file * @param bytes - * the size of the file file in bytes + * the size of the file in bytes * @param digest * the digest information of the file * @param origin diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileId.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileId.scala index 0af0c34bd0..a55f658f1f 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileId.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileId.scala @@ -19,6 +19,7 @@ object FileId { def apply(ref: ResourceRef, project: ProjectRef): FileId = FileId(IdSegmentRef(ref), project) def apply(id: IdSegment, tag: UserTag, project: ProjectRef): FileId = FileId(IdSegmentRef(id, tag), project) def apply(id: IdSegment, rev: Int, project: ProjectRef): FileId = FileId(IdSegmentRef(id, rev), project) + def apply(id: IdSegment, project: ProjectRef): FileId = FileId(IdSegmentRef(id), project) val iriExpander: ExpandIri[InvalidFileId] = new ExpandIri(InvalidFileId.apply) } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileRejection.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileRejection.scala index 2c6a60f187..4fb4909c6d 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileRejection.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/model/FileRejection.scala @@ -8,7 +8,7 @@ import ch.epfl.bluebrain.nexus.delta.kernel.utils.ClassUtils import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageRejection import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageRejection.StorageFetchRejection import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.{FetchFileRejection, SaveFileRejection} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.{CopyFileRejection, FetchFileRejection, SaveFileRejection} import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.ContextValue @@ -235,6 +235,16 @@ object FileRejection { final case class LinkRejection(id: Iri, storageId: Iri, rejection: StorageFileRejection) extends FileRejection(s"File '$id' could not be linked using storage '$storageId'", Some(rejection.loggedDetails)) + final case class CopyRejection( + sourceProj: ProjectRef, + destProject: ProjectRef, + destStorageId: Iri, + rejection: CopyFileRejection + ) extends FileRejection( + s"Failed to copy files from $sourceProj to storage $destStorageId in project $destProject", + Some(rejection.loggedDetails) + ) + /** * Signals a rejection caused when interacting with other APIs when fetching a resource */ @@ -259,6 +269,8 @@ object FileRejection { obj.add(keywords.tpe, ClassUtils.simpleName(rejection).asJson).add("details", rejection.loggedDetails.asJson) case LinkRejection(_, _, rejection) => obj.add(keywords.tpe, ClassUtils.simpleName(rejection).asJson).add("details", rejection.loggedDetails.asJson) + case CopyRejection(_, _, _, rejection) => + obj.add(keywords.tpe, ClassUtils.simpleName(rejection).asJson).add("details", rejection.loggedDetails.asJson) case ProjectContextRejection(rejection) => rejection.asJsonObject case IncorrectRev(provided, expected) => obj.add("provided", provided.asJson).add("expected", expected.asJson) case _: FileNotFound => obj.add(keywords.tpe, "ResourceNotFound".asJson) @@ -283,6 +295,7 @@ object FileRejection { // If this happens it signifies a system problem rather than the user having made a mistake case FetchRejection(_, _, FetchFileRejection.FileNotFound(_)) => (StatusCodes.InternalServerError, Seq.empty) case SaveRejection(_, _, SaveFileRejection.ResourceAlreadyExists(_)) => (StatusCodes.Conflict, Seq.empty) + case CopyRejection(_, _, _, rejection) => (rejection.status, Seq.empty) case FetchRejection(_, _, _) => (StatusCodes.InternalServerError, Seq.empty) case SaveRejection(_, _, _) => (StatusCodes.InternalServerError, Seq.empty) case _ => (StatusCodes.BadRequest, Seq.empty) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesRoutes.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesRoutes.scala new file mode 100644 index 0000000000..a85d653199 --- /dev/null +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesRoutes.scala @@ -0,0 +1,103 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes + +import akka.http.scaladsl.model.StatusCodes.Created +import akka.http.scaladsl.server._ +import cats.data.EitherT +import cats.effect.IO +import cats.syntax.all._ +import ch.epfl.bluebrain.nexus.delta.kernel.Logger +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch.BatchFiles +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{CopyFileDestination, File, FileRejection} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.permissions.{read => Read} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{contexts, FileResource} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StoragesConfig.StorageTypeConfig +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.{ContextValue, RemoteContextResolution} +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.encoder.JsonLdEncoder +import ch.epfl.bluebrain.nexus.delta.rdf.utils.JsonKeyOrdering +import ch.epfl.bluebrain.nexus.delta.sdk._ +import ch.epfl.bluebrain.nexus.delta.sdk.acls.AclCheck +import ch.epfl.bluebrain.nexus.delta.sdk.circe.CirceUnmarshalling +import ch.epfl.bluebrain.nexus.delta.sdk.directives.DeltaDirectives._ +import ch.epfl.bluebrain.nexus.delta.sdk.directives.{AuthDirectives, DeltaSchemeDirectives} +import ch.epfl.bluebrain.nexus.delta.sdk.error.ServiceError.AuthorizationFailed +import ch.epfl.bluebrain.nexus.delta.sdk.identities.Identities +import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller +import ch.epfl.bluebrain.nexus.delta.sdk.jsonld.BulkOperationResults +import ch.epfl.bluebrain.nexus.delta.sdk.model.{BaseUri, IdSegment} + +final class BatchFilesRoutes( + identities: Identities, + aclCheck: AclCheck, + batchFiles: BatchFiles, + schemeDirectives: DeltaSchemeDirectives, + index: IndexingAction.Execute[File] +)(implicit + baseUri: BaseUri, + storageConfig: StorageTypeConfig, + cr: RemoteContextResolution, + ordering: JsonKeyOrdering +) extends AuthDirectives(identities, aclCheck) + with CirceUnmarshalling { self => + + private val logger = Logger[BatchFilesRoutes] + + import schemeDirectives.resolveProjectRef + + implicit val bulkOpJsonLdEnc: JsonLdEncoder[BulkOperationResults[FileResource]] = + BulkOperationResults.searchResultsJsonLdEncoder(ContextValue(contexts.files)) + + def routes: Route = + baseUriPrefix(baseUri.prefix) { + pathPrefix("bulk") { + pathPrefix("files") { + extractCaller { implicit caller => + resolveProjectRef.apply { projectRef => + (post & pathEndOrSingleSlash & parameter("storage".as[IdSegment].?) & indexingMode & tagParam) { + (storage, mode, tag) => + // Bulk create files by copying from another project + entity(as[CopyFileSource]) { c: CopyFileSource => + val copyTo = CopyFileDestination(projectRef, storage, tag) + emit(Created, copyFiles(mode, c, copyTo)) + } + } + } + } + } + } + } + + private def copyFiles(mode: IndexingMode, source: CopyFileSource, dest: CopyFileDestination)(implicit + caller: Caller + ): IO[Either[FileRejection, BulkOperationResults[FileResource]]] = + (for { + _ <- + EitherT.right(aclCheck.authorizeForOr(source.project, Read)(AuthorizationFailed(source.project.project, Read))) + results <- EitherT(batchFiles.copyFiles(source, dest).attemptNarrow[FileRejection]) + _ <- EitherT.right[FileRejection](results.traverse(index(dest.project, _, mode))) + _ <- EitherT.right[FileRejection](logger.info(s"Bulk file copy succeeded with results: $results")) + } yield BulkOperationResults(results.toList)) + .onError(e => + EitherT.right(logger.error(e)(s"Bulk file copy operation failed for source $source and destination $dest")) + ) + .value +} + +object BatchFilesRoutes { + + def apply( + config: StorageTypeConfig, + identities: Identities, + aclCheck: AclCheck, + batchFiles: BatchFiles, + schemeDirectives: DeltaSchemeDirectives, + index: IndexingAction.Execute[File] + )(implicit + baseUri: BaseUri, + cr: RemoteContextResolution, + ordering: JsonKeyOrdering + ): Route = { + implicit val storageTypeConfig: StorageTypeConfig = config + new BatchFilesRoutes(identities, aclCheck, batchFiles, schemeDirectives, index).routes + } + +} diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/CopyFileSource.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/CopyFileSource.scala new file mode 100644 index 0000000000..c259fa68e8 --- /dev/null +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/CopyFileSource.scala @@ -0,0 +1,42 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes + +import cats.data.NonEmptyList +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileId +import ch.epfl.bluebrain.nexus.delta.sdk.model.IdSegment +import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef +import ch.epfl.bluebrain.nexus.delta.sourcing.model.Tag.UserTag +import io.circe.{Decoder, DecodingFailure, Json} + +final case class CopyFileSource( + project: ProjectRef, + files: NonEmptyList[FileId] +) + +object CopyFileSource { + + implicit val dec: Decoder[CopyFileSource] = Decoder.instance { cur => + def parseSingle(j: Json, proj: ProjectRef): Decoder.Result[FileId] = + for { + sourceFile <- j.hcursor.get[String]("sourceFileId").map(IdSegment(_)) + sourceTag <- j.hcursor.get[Option[UserTag]]("sourceTag") + sourceRev <- j.hcursor.get[Option[Int]]("sourceRev") + fileId <- parseFileId(sourceFile, proj, sourceTag, sourceRev) + } yield fileId + + def parseFileId(id: IdSegment, proj: ProjectRef, sourceTag: Option[UserTag], sourceRev: Option[Int]) = + (sourceTag, sourceRev) match { + case (Some(tag), None) => Right(FileId(id, tag, proj)) + case (None, Some(rev)) => Right(FileId(id, rev, proj)) + case (None, None) => Right(FileId(id, proj)) + case (Some(_), Some(_)) => + Left( + DecodingFailure("Tag and revision cannot be simultaneously present for source file lookup", Nil) + ) + } + + for { + sourceProj <- cur.get[ProjectRef]("sourceProjectRef") + files <- cur.get[NonEmptyList[Json]]("files").flatMap(_.traverse(parseSingle(_, sourceProj))) + } yield CopyFileSource(sourceProj, files) + } +} diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutes.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutes.scala index 7686829d3a..815ca13d34 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutes.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutes.scala @@ -7,6 +7,7 @@ import akka.http.scaladsl.model.{ContentType, MediaRange} import akka.http.scaladsl.server._ import cats.effect.IO import cats.syntax.all._ + import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{File, FileId, FileRejection} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.permissions.{read => Read, write => Write} @@ -25,6 +26,7 @@ import ch.epfl.bluebrain.nexus.delta.sdk.fusion.FusionConfig import ch.epfl.bluebrain.nexus.delta.sdk.identities.Identities import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller import ch.epfl.bluebrain.nexus.delta.sdk.implicits._ + import ch.epfl.bluebrain.nexus.delta.sdk.model.routes.Tag import ch.epfl.bluebrain.nexus.delta.sdk.model.{BaseUri, IdSegment} import ch.epfl.bluebrain.nexus.delta.sourcing.model.Tag.UserTag @@ -109,8 +111,31 @@ final class FilesRoutes( operationName(s"$prefixSegment/files/{org}/{project}/{id}") { concat( (put & pathEndOrSingleSlash) { - parameters("rev".as[Int].?, "storage".as[IdSegment].?, "tag".as[UserTag].?) { - case (None, storage, tag) => + concat( + parameters("rev".as[Int], "storage".as[IdSegment].?, "tag".as[UserTag].?) { + case (rev, storage, tag) => + concat( + // Update a Link + entity(as[LinkFile]) { case LinkFile(filename, mediaType, path) => + emit( + files + .updateLink(fileId, storage, filename, mediaType, path, rev, tag) + .index(mode) + .attemptNarrow[FileRejection] + ) + }, + // Update a file + extractRequestEntity { entity => + emit( + files + .update(fileId, storage, rev, entity, tag) + .index(mode) + .attemptNarrow[FileRejection] + ) + } + ) + }, + parameters("storage".as[IdSegment].?, "tag".as[UserTag].?) { case (storage, tag) => concat( // Link a file with id segment entity(as[LinkFile]) { case LinkFile(filename, mediaType, path) => @@ -126,32 +151,15 @@ final class FilesRoutes( extractRequestEntity { entity => emit( Created, - files.create(fileId, storage, entity, tag).index(mode).attemptNarrow[FileRejection] - ) - } - ) - case (Some(rev), storage, tag) => - concat( - // Update a Link - entity(as[LinkFile]) { case LinkFile(filename, mediaType, path) => - emit( - files - .updateLink(fileId, storage, filename, mediaType, path, rev, tag) - .index(mode) - .attemptNarrow[FileRejection] - ) - }, - // Update a file - extractRequestEntity { entity => - emit( files - .update(fileId, storage, rev, entity, tag) + .create(fileId, storage, entity, tag) .index(mode) .attemptNarrow[FileRejection] ) } ) - } + } + ) }, // Deprecate a file (delete & parameter("rev".as[Int])) { rev => @@ -173,7 +181,7 @@ final class FilesRoutes( ) } }, - (pathPrefix("tags")) { + pathPrefix("tags") { operationName(s"$prefixSegment/files/{org}/{project}/{id}/tags") { concat( // Fetch a file tags diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/FetchStorage.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/FetchStorage.scala new file mode 100644 index 0000000000..d66b9009c6 --- /dev/null +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/FetchStorage.scala @@ -0,0 +1,45 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages + +import cats.effect.IO +import cats.implicits.catsSyntaxMonadError +import ch.epfl.bluebrain.nexus.delta.kernel.Mapper +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageRejection.StorageFetchRejection +import ch.epfl.bluebrain.nexus.delta.sdk.model.IdSegmentRef +import ch.epfl.bluebrain.nexus.delta.sourcing.model.{ProjectRef, ResourceRef} + +trait FetchStorage { + + /** + * Fetch the storage using the ''resourceRef'' + * + * @param resourceRef + * the storage reference (Latest, Revision or Tag) + * @param project + * the project where the storage belongs + */ + final def fetch[R <: Throwable]( + resourceRef: ResourceRef, + project: ProjectRef + )(implicit rejectionMapper: Mapper[StorageFetchRejection, R]): IO[StorageResource] = + fetch(IdSegmentRef(resourceRef), project).adaptError { case err: StorageFetchRejection => + rejectionMapper.to(err) + } + + /** + * Fetch the last version of a storage + * + * @param id + * the identifier that will be expanded to the Iri of the storage with its optional rev/tag + * @param project + * the project where the storage belongs + */ + def fetch(id: IdSegmentRef, project: ProjectRef): IO[StorageResource] + + /** + * Fetches the default storage for a project. + * + * @param project + * the project where to look for the default storage + */ + def fetchDefault(project: ProjectRef): IO[StorageResource] +} diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/Storages.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/Storages.scala index ffa4a9ef17..b1ecf85514 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/Storages.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/Storages.scala @@ -2,9 +2,9 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages import cats.effect.{Clock, IO} import cats.syntax.all._ +import ch.epfl.bluebrain.nexus.delta.kernel.Logger import ch.epfl.bluebrain.nexus.delta.kernel.kamon.KamonMetricComponent import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF -import ch.epfl.bluebrain.nexus.delta.kernel.{Logger, Mapper} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.Storages._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StoragesConfig.StorageTypeConfig import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageCommand._ @@ -30,7 +30,7 @@ import ch.epfl.bluebrain.nexus.delta.sourcing.ScopedEntityDefinition.Tagger import ch.epfl.bluebrain.nexus.delta.sourcing._ import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Subject import ch.epfl.bluebrain.nexus.delta.sourcing.model.Tag.UserTag -import ch.epfl.bluebrain.nexus.delta.sourcing.model.{EntityType, ProjectRef, ResourceRef} +import ch.epfl.bluebrain.nexus.delta.sourcing.model.{EntityType, ProjectRef} import ch.epfl.bluebrain.nexus.delta.sourcing.stream.Elem import fs2.Stream import io.circe.Json @@ -46,7 +46,7 @@ final class Storages private ( fetchContext: FetchContext[StorageFetchRejection], sourceDecoder: JsonLdSourceResolvingDecoder[StorageRejection, StorageFields], serviceAccount: ServiceAccount -) { +) extends FetchStorage { implicit private val kamonComponent: KamonMetricComponent = KamonMetricComponent(entityType.value) @@ -255,31 +255,7 @@ final class Storages private ( } yield res }.span("undeprecateStorage") - /** - * Fetch the storage using the ''resourceRef'' - * - * @param resourceRef - * the storage reference (Latest, Revision or Tag) - * @param project - * the project where the storage belongs - */ - def fetch[R <: Throwable]( - resourceRef: ResourceRef, - project: ProjectRef - )(implicit rejectionMapper: Mapper[StorageFetchRejection, R]): IO[StorageResource] = - fetch(IdSegmentRef(resourceRef), project).adaptError { case err: StorageFetchRejection => - rejectionMapper.to(err) - } - - /** - * Fetch the last version of a storage - * - * @param id - * the identifier that will be expanded to the Iri of the storage with its optional rev/tag - * @param project - * the project where the storage belongs - */ - def fetch(id: IdSegmentRef, project: ProjectRef): IO[StorageResource] = { + override def fetch(id: IdSegmentRef, project: ProjectRef): IO[StorageResource] = { for { pc <- fetchContext.onRead(project) iri <- expandIri(id.value, pc) @@ -299,13 +275,7 @@ final class Storages private ( .currentStates(Scope.Project(project), _.toResource) .filter(_.value.default) - /** - * Fetches the default storage for a project. - * - * @param project - * the project where to look for the default storage - */ - def fetchDefault(project: ProjectRef): IO[StorageResource] = { + override def fetchDefault(project: ProjectRef): IO[StorageResource] = { for { defaultOpt <- fetchDefaults(project).reduce(updatedByDesc.min(_, _)).head.compile.last default <- IO.fromOption(defaultOpt)(DefaultStorageNotFound(project)) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StoragesStatistics.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StoragesStatistics.scala index e047e2c92e..84f11822dd 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StoragesStatistics.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StoragesStatistics.scala @@ -4,7 +4,7 @@ import akka.http.scaladsl.model.Uri.Query import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.EventMetricsProjection.eventMetricsIndex import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.client.ElasticSearchClient -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageStatEntry +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{Storage, StorageStatEntry} import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.sdk.model.IdSegment import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef @@ -18,6 +18,17 @@ trait StoragesStatistics { */ def get(idSegment: IdSegment, project: ProjectRef): IO[StorageStatEntry] + /** + * Retrieve remaining space on a storage if it has a capacity. + */ + final def getStorageAvailableSpace(storage: Storage): IO[Option[Long]] = + storage.storageValue.capacity.fold(IO.none[Long]) { capacity => + get(storage.id, storage.project) + .redeem( + _ => Some(capacity), + stat => Some(capacity - stat.spaceUsed) + ) + } } object StoragesStatistics { diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/Storage.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/Storage.scala index 35a256462b..b31392f729 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/Storage.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/Storage.scala @@ -4,11 +4,11 @@ import akka.actor.ActorSystem import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StoragesConfig.StorageTypeConfig import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.Metadata import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageValue.{DiskStorageValue, RemoteDiskStorageValue, S3StorageValue} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.{DiskStorageFetchFile, DiskStorageSaveFile} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.RemoteDiskStorageClient import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.s3.{S3StorageFetchFile, S3StorageLinkFile, S3StorageSaveFile} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.{FetchAttributes, FetchFile, LinkFile, SaveFile} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{contexts, Storages} import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.ContextValue @@ -88,7 +88,6 @@ object Storage { def saveFile(implicit as: ActorSystem): SaveFile = new DiskStorageSaveFile(this) - } /** diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/StorageRejection.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/StorageRejection.scala index d41cf19bc6..271d0672bd 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/StorageRejection.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/model/StorageRejection.scala @@ -143,10 +143,11 @@ object StorageRejection { extends StorageRejection(s"Storage ${id.fold("")(id => s"'$id'")} has invalid JSON-LD payload.") /** - * Rejection returned when attempting to create a storage with an id that already exists. + * Signals an attempt to update a storage to a different storage type * * @param id - * the storage identifier + * @param found + * @param expected */ final case class DifferentStorageType(id: Iri, found: StorageType, expected: StorageType) extends StorageRejection(s"Storage '$id' is of type '$found' and can't be updated to be a '$expected' .") diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/StorageFileRejection.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/StorageFileRejection.scala index b8f6122b90..8e7036481a 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/StorageFileRejection.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/StorageFileRejection.scala @@ -1,7 +1,11 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageType +import akka.http.scaladsl.model.StatusCodes import ch.epfl.bluebrain.nexus.delta.kernel.error.Rejection +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageType +import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri +import ch.epfl.bluebrain.nexus.delta.sdk.http.HttpClientError +import ch.epfl.bluebrain.nexus.delta.sdk.marshalling.HttpResponseFields /** * Enumeration of Storage rejections related to file operations. @@ -65,6 +69,43 @@ object StorageFileRejection { extends FetchAttributeRejection(rejection.loggedDetails) } + sealed abstract class CopyFileRejection(loggedDetails: String) extends StorageFileRejection(loggedDetails) + + object CopyFileRejection { + final case class UnsupportedOperation(tpe: StorageType) + extends CopyFileRejection( + s"Copying a file attributes is not supported for storages of type '${tpe.iri}'" + ) + + final case class SourceFileTooLarge(maxSize: Long, storageId: Iri) + extends CopyFileRejection( + s"Source file size exceeds maximum $maxSize on destination storage $storageId" + ) + + final case class TotalCopySizeTooLarge(totalSize: Long, spaceLeft: Long, storageId: Iri) + extends CopyFileRejection( + s"Combined size of source files ($totalSize) exceeds space ($spaceLeft) on destination storage $storageId" + ) + + final case class RemoteDiskClientError(underlying: HttpClientError) + extends CopyFileRejection( + s"Error from remote disk storage client: ${underlying.asString}" + ) + + final case class DifferentStorageTypes(id: Iri, source: StorageType, dest: StorageType) + extends CopyFileRejection( + s"Source storage $id of type $source cannot be different to the destination storage type $dest" + ) + + implicit val statusCodes: HttpResponseFields[CopyFileRejection] = HttpResponseFields { + case _: UnsupportedOperation => StatusCodes.BadRequest + case _: SourceFileTooLarge => StatusCodes.BadRequest + case _: TotalCopySizeTooLarge => StatusCodes.BadRequest + case _: DifferentStorageTypes => StatusCodes.BadRequest + case _: RemoteDiskClientError => StatusCodes.InternalServerError + } + } + /** * Rejection returned when a storage cannot save a file */ diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskCopyDetails.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskCopyDetails.scala new file mode 100644 index 0000000000..182efab2ee --- /dev/null +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskCopyDetails.scala @@ -0,0 +1,10 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk + +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes, FileDescription} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.DiskStorage + +final case class DiskCopyDetails( + destStorage: DiskStorage, + destinationDesc: FileDescription, + sourceAttributes: FileAttributes +) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageCopyFiles.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageCopyFiles.scala new file mode 100644 index 0000000000..8218d4b715 --- /dev/null +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageCopyFiles.scala @@ -0,0 +1,55 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk + +import akka.http.scaladsl.model.Uri +import cats.data.NonEmptyList +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.kernel.utils.{CopyBetween, TransactionalFileCopier} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.DiskStorage +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.DiskStorageSaveFile.computeLocation +import fs2.io.file.Path + +import java.nio.file + +trait DiskStorageCopyFiles { + def copyFiles(destStorage: DiskStorage, details: NonEmptyList[DiskCopyDetails]): IO[NonEmptyList[FileAttributes]] +} + +object DiskStorageCopyFiles { + def mk(copier: TransactionalFileCopier): DiskStorageCopyFiles = new DiskStorageCopyFiles { + + def copyFiles(destStorage: DiskStorage, details: NonEmptyList[DiskCopyDetails]): IO[NonEmptyList[FileAttributes]] = + details + .traverse(mkCopyDetailsAndDestAttributes(destStorage, _)) + .flatMap { copyDetailsAndDestAttributes => + val copyDetails = copyDetailsAndDestAttributes.map(_._1) + val destAttrs = copyDetailsAndDestAttributes.map(_._2) + copier.copyAll(copyDetails).as(destAttrs) + } + + private def mkCopyDetailsAndDestAttributes(destStorage: DiskStorage, copyFile: DiskCopyDetails) = + for { + sourcePath <- absoluteDiskPathFromAttributes(copyFile.sourceAttributes) + (destPath, destRelativePath) <- computeDestLocation(destStorage, copyFile) + destAttr = mkDestAttributes(copyFile, destPath, destRelativePath) + copyDetails <- absoluteDiskPathFromAttributes(destAttr).map { dest => + CopyBetween(Path.fromNioPath(sourcePath), Path.fromNioPath(dest)) + } + } yield (copyDetails, destAttr) + + private def computeDestLocation(destStorage: DiskStorage, cd: DiskCopyDetails) = + computeLocation(destStorage.project, destStorage.value, cd.destinationDesc.uuid, cd.destinationDesc.filename) + + private def mkDestAttributes(cd: DiskCopyDetails, destPath: file.Path, destRelativePath: file.Path) = + FileAttributes( + uuid = cd.destinationDesc.uuid, + location = Uri(destPath.toUri.toString), + path = Uri.Path(destRelativePath.toString), + filename = cd.destinationDesc.filename, + mediaType = cd.sourceAttributes.mediaType, + bytes = cd.sourceAttributes.bytes, + digest = cd.sourceAttributes.digest, + origin = cd.sourceAttributes.origin + ) + } +} diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageFetchFile.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageFetchFile.scala index 11d793d83e..2bc3943790 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageFetchFile.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageFetchFile.scala @@ -9,21 +9,18 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.Storage import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.FetchFileRejection.UnexpectedLocationFormat import ch.epfl.bluebrain.nexus.delta.sdk.AkkaSource -import java.net.URI -import java.nio.file.Paths -import scala.util.{Failure, Success, Try} - object DiskStorageFetchFile extends FetchFile { override def apply(attributes: FileAttributes): IO[AkkaSource] = apply(attributes.location.path) override def apply(path: Uri.Path): IO[AkkaSource] = - Try(Paths.get(URI.create(s"file://$path"))) match { - case Failure(err) => IO.raiseError(UnexpectedLocationFormat(s"file://$path", err.getMessage)) - case Success(path) => - IO.raiseWhen(!path.toFile.exists())(FetchFileRejection.FileNotFound(path.toString)) >> IO.blocking( - FileIO.fromPath(path) - ) - } + absoluteDiskPath(path).redeemWith( + e => IO.raiseError(UnexpectedLocationFormat(s"file://$path", e.getMessage)), + path => + IO.blocking(path.toFile.exists()).flatMap { exists => + if (exists) IO.blocking(FileIO.fromPath(path)) + else IO.raiseError(FetchFileRejection.FileNotFound(path.toString)) + } + ) } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageSaveFile.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageSaveFile.scala index cae5520aae..2172233418 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageSaveFile.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/DiskStorageSaveFile.scala @@ -70,13 +70,23 @@ object DiskStorageSaveFile { disk: DiskStorageValue, uuid: UUID, filename: String + ): IO[(Path, Path)] = + for { + (resolved, relative) <- computeLocation(project, disk, uuid, filename) + dir = resolved.getParent + _ <- IO.blocking(Files.createDirectories(dir)).adaptError(couldNotCreateDirectory(dir, _)) + } yield resolved -> relative + + def computeLocation( + project: ProjectRef, + disk: DiskStorageValue, + uuid: UUID, + filename: String ): IO[(Path, Path)] = { val relativePath = intermediateFolders(project, uuid, filename) for { relative <- IO.delay(Paths.get(relativePath)).adaptError(wrongPath(relativePath, _)) resolved <- IO.delay(disk.volume.value.resolve(relative)).adaptError(wrongPath(relativePath, _)) - dir = resolved.getParent - _ <- IO.delay(Files.createDirectories(dir)).adaptError(couldNotCreateDirectory(dir, _)) } yield resolved -> relative } diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/package.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/package.scala new file mode 100644 index 0000000000..996e861fc4 --- /dev/null +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/disk/package.scala @@ -0,0 +1,17 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations + +import akka.http.scaladsl.model.Uri.Path +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes + +import java.net.URI +import java.nio.file +import java.nio.file.Paths + +package object disk { + + def absoluteDiskPathFromAttributes(attr: FileAttributes): IO[file.Path] = absoluteDiskPath(attr.location.path) + + def absoluteDiskPath(relative: Path): IO[file.Path] = IO(Paths.get(URI.create(s"file://$relative"))) + +} diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageCopyFiles.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageCopyFiles.scala new file mode 100644 index 0000000000..3d84070755 --- /dev/null +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/RemoteDiskStorageCopyFiles.scala @@ -0,0 +1,60 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote + +import akka.http.scaladsl.model.Uri +import akka.http.scaladsl.model.Uri.Path +import cats.data.NonEmptyList +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.RemoteDiskStorage +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.SaveFile.intermediateFolders +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.RemoteDiskStorageClient +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.model.{RemoteDiskCopyDetails, RemoteDiskCopyPaths} + +trait RemoteDiskStorageCopyFiles { + def copyFiles( + destStorage: RemoteDiskStorage, + copyDetails: NonEmptyList[RemoteDiskCopyDetails] + ): IO[NonEmptyList[FileAttributes]] +} + +object RemoteDiskStorageCopyFiles { + + def mk(client: RemoteDiskStorageClient): RemoteDiskStorageCopyFiles = new RemoteDiskStorageCopyFiles { + def copyFiles( + destStorage: RemoteDiskStorage, + copyDetails: NonEmptyList[RemoteDiskCopyDetails] + ): IO[NonEmptyList[FileAttributes]] = { + + val paths = remoteDiskCopyPaths(destStorage, copyDetails) + + client.copyFiles(destStorage.value.folder, paths)(destStorage.value.endpoint).map { destPaths => + copyDetails.zip(paths).zip(destPaths).map { case ((copyDetails, remoteCopyPaths), absoluteDestPath) => + mkDestAttributes(copyDetails, remoteCopyPaths.destPath, absoluteDestPath) + } + } + } + } + + private def mkDestAttributes(cd: RemoteDiskCopyDetails, relativeDestPath: Path, absoluteDestPath: Uri) = { + val destDesc = cd.destinationDesc + val sourceAttr = cd.sourceAttributes + FileAttributes( + uuid = destDesc.uuid, + location = absoluteDestPath, + path = relativeDestPath, + filename = destDesc.filename, + mediaType = destDesc.mediaType, + bytes = sourceAttr.bytes, + digest = sourceAttr.digest, + origin = sourceAttr.origin + ) + } + + private def remoteDiskCopyPaths(destStorage: RemoteDiskStorage, copyDetails: NonEmptyList[RemoteDiskCopyDetails]) = + copyDetails.map { cd => + val destDesc = cd.destinationDesc + val destinationPath = Uri.Path(intermediateFolders(destStorage.project, destDesc.uuid, destDesc.filename)) + val sourcePath = cd.sourceAttributes.path + RemoteDiskCopyPaths(cd.sourceBucket, sourcePath, destinationPath) + } +} diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/RemoteDiskStorageClient.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/RemoteDiskStorageClient.scala index a14bb20a2c..5aac781cfb 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/RemoteDiskStorageClient.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/RemoteDiskStorageClient.scala @@ -2,17 +2,19 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote import akka.actor.ActorSystem import akka.http.scaladsl.client.RequestBuilding._ -import akka.http.scaladsl.model.BodyPartEntity import akka.http.scaladsl.model.Multipart.FormData import akka.http.scaladsl.model.Multipart.FormData.BodyPart import akka.http.scaladsl.model.StatusCodes._ import akka.http.scaladsl.model.Uri.Path +import akka.http.scaladsl.model.{BodyPartEntity, Uri} +import cats.data.NonEmptyList import cats.effect.IO import cats.implicits._ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.FetchFileRejection.UnexpectedFetchError import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.MoveFileRejection.UnexpectedMoveError -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.{FetchFileRejection, MoveFileRejection, SaveFileRejection} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.model.RemoteDiskStorageFileAttributes +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.{CopyFileRejection, FetchFileRejection, MoveFileRejection, SaveFileRejection} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.model.{RemoteDiskCopyPaths, RemoteDiskStorageFileAttributes} +import ch.epfl.bluebrain.nexus.delta.rdf.implicits.uriDecoder import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.JsonLdContext.keywords import ch.epfl.bluebrain.nexus.delta.sdk.AkkaSource import ch.epfl.bluebrain.nexus.delta.sdk.auth.{AuthTokenProvider, Credentials} @@ -175,6 +177,30 @@ final class RemoteDiskStorageClient(client: HttpClient, getAuthToken: AuthTokenP } } + /** + * Copies files to a destination bucket. Source files can be located within different buckets. File attributes are + * not recomputed, so only the file paths will change. + * + * If any copies fail the whole operation will be aborted and the remote storage service will return an error. + * + * @return + * Absolute locations of the created files, preserving the input order. + */ + def copyFiles( + destBucket: Label, + files: NonEmptyList[RemoteDiskCopyPaths] + )(implicit baseUri: BaseUri): IO[NonEmptyList[Uri]] = + getAuthToken(credentials).flatMap { authToken => + val endpoint = baseUri.endpoint / "buckets" / destBucket.value / "files" + + implicit val dec: Decoder[NonEmptyList[Uri]] = Decoder[NonEmptyList[Json]].emap { nel => + nel.traverse(_.hcursor.get[Uri]("absoluteDestinationLocation").leftMap(_.toString())) + } + client + .fromJsonTo[NonEmptyList[Uri]](Post(endpoint, files.asJson).withCredentials(authToken)) + .adaptError { case error: HttpClientError => CopyFileRejection.RemoteDiskClientError(error) } + } + private def bucketNotFoundType(error: HttpClientError): Boolean = error.jsonBody.fold(false)(_.hcursor.get[String](keywords.tpe).toOption.contains("BucketNotFound")) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/model/RemoteDiskCopyDetails.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/model/RemoteDiskCopyDetails.scala new file mode 100644 index 0000000000..1ad264e161 --- /dev/null +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/model/RemoteDiskCopyDetails.scala @@ -0,0 +1,12 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.model + +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes, FileDescription} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.RemoteDiskStorage +import ch.epfl.bluebrain.nexus.delta.sourcing.model.Label + +final case class RemoteDiskCopyDetails( + destStorage: RemoteDiskStorage, + destinationDesc: FileDescription, + sourceBucket: Label, + sourceAttributes: FileAttributes +) diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/model/RemoteDiskCopyPaths.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/model/RemoteDiskCopyPaths.scala new file mode 100644 index 0000000000..c19726b430 --- /dev/null +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/model/RemoteDiskCopyPaths.scala @@ -0,0 +1,19 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.model + +import akka.http.scaladsl.model.Uri.Path +import ch.epfl.bluebrain.nexus.delta.sourcing.model.Label +import io.circe.syntax.KeyOps +import io.circe.{Encoder, Json} + +final case class RemoteDiskCopyPaths( + sourceBucket: Label, + sourcePath: Path, + destPath: Path +) + +object RemoteDiskCopyPaths { + implicit val enc: Encoder[RemoteDiskCopyPaths] = Encoder.instance { + case RemoteDiskCopyPaths(sourceBucket, source, dest) => + Json.obj("sourceBucket" := sourceBucket, "source" := source.toString(), "destination" := dest.toString()) + } +} diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/routes/StoragesRoutes.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/routes/StoragesRoutes.scala index b7524dfc99..fad4ae94ea 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/routes/StoragesRoutes.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/routes/StoragesRoutes.scala @@ -60,7 +60,7 @@ final class StoragesRoutes( (baseUriPrefix(baseUri.prefix) & replaceUri("storages", schemas.storage)) { pathPrefix("storages") { extractCaller { implicit caller => - resolveProjectRef.apply { implicit ref => + resolveProjectRef.apply { ref => concat( (pathEndOrSingleSlash & operationName(s"$prefixSegment/storages/{org}/{project}")) { // Create a storage without id segment diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FileFixtures.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FileFixtures.scala index 729dd5e4f8..b66d2c4e96 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FileFixtures.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FileFixtures.scala @@ -1,72 +1,56 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.files import akka.http.scaladsl.model.ContentTypes.`text/plain(UTF-8)` -import akka.http.scaladsl.model.{HttpEntity, MessageEntity, Multipart, Uri} +import akka.http.scaladsl.model.{HttpEntity, MessageEntity, Multipart} +import cats.effect.unsafe.implicits.global import cats.effect.{IO, Ref} import ch.epfl.bluebrain.nexus.delta.kernel.utils.{UUIDF, UrlUtils} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest.ComputedDigest +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.generators.FileGen import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes.FileAttributesOrigin.Client -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{AbsolutePath, DigestAlgorithm} import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.nxv import ch.epfl.bluebrain.nexus.delta.sdk.generators.ProjectGen import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.ApiMappings -import ch.epfl.bluebrain.nexus.delta.sourcing.model.Label -import ch.epfl.bluebrain.nexus.testkit.scalatest.EitherValues -import ch.epfl.bluebrain.nexus.testkit.scalatest.ce.CatsIOValues -import org.scalatest.Suite +import ch.epfl.bluebrain.nexus.delta.sourcing.model.{Label, ProjectRef} -import java.nio.file.{Files => JavaFiles} import java.util.{Base64, UUID} -trait FileFixtures extends EitherValues with CatsIOValues { +trait FileFixtures { - self: Suite => - - val uuid = UUID.fromString("8249ba90-7cc6-4de5-93a1-802c04200dcc") - val uuid2 = UUID.fromString("12345678-7cc6-4de5-93a1-802c04200dcc") - val ref = Ref.of[IO, UUID](uuid).accepted - implicit val uuidF: UUIDF = UUIDF.fromRef(ref) - val org = Label.unsafe("org") - val orgDeprecated = Label.unsafe("org-deprecated") - val project = ProjectGen.project("org", "proj", base = nxv.base, mappings = ApiMappings("file" -> schemas.files)) - val deprecatedProject = ProjectGen.project("org", "proj-deprecated") - val projectWithDeprecatedOrg = ProjectGen.project("org-deprecated", "other-proj") - val projectRef = project.ref - val diskId2 = nxv + "disk2" - val file1 = nxv + "file1" - val file2 = nxv + "file2" - val fileTagged = nxv + "fileTagged" - val fileTagged2 = nxv + "fileTagged2" - val file1Encoded = UrlUtils.encode(file1.toString) - val encodeId = (id: String) => UrlUtils.encode((nxv + id).toString) - val generatedId = project.base.iri / uuid.toString - val generatedId2 = project.base.iri / uuid2.toString + val uuid = UUID.fromString("8249ba90-7cc6-4de5-93a1-802c04200dcc") + val uuid2 = UUID.fromString("12345678-7cc6-4de5-93a1-802c04200dcc") + val uuidOrg2 = UUID.fromString("66666666-7cc6-4de5-93a1-802c04200dcc") + val ref = Ref.of[IO, UUID](uuid).unsafeRunSync() + implicit val uuidF: UUIDF = UUIDF.fromRef(ref) + val org = Label.unsafe("org") + val org2 = Label.unsafe("org2") + val project = ProjectGen.project(org.value, "proj", base = nxv.base, mappings = ApiMappings("file" -> schemas.files)) + val deprecatedProject = ProjectGen.project("org", "proj-deprecated") + val projectRef = project.ref + val diskId2 = nxv + "disk2" + val file1 = nxv + "file1" + val file2 = nxv + "file2" + val fileTagged = nxv + "fileTagged" + val fileTagged2 = nxv + "fileTagged2" + val file1Encoded = UrlUtils.encode(file1.toString) + val encodeId = (id: String) => UrlUtils.encode((nxv + id).toString) + val generatedId = project.base.iri / uuid.toString + val generatedId2 = project.base.iri / uuid2.toString val content = "file content" - val path = AbsolutePath(JavaFiles.createTempDirectory("files")).rightValue - val digest = - ComputedDigest(DigestAlgorithm.default, "e0ac3601005dfa1864f5392aabaf7d898b1b5bab854f1acb4491bcd806b76b0c") + val path = FileGen.mkTempDir("files") def withUUIDF[T](id: UUID)(test: => T): T = (for { old <- ref.getAndSet(id) - t <- IO.delay(test).onError(_ => ref.set(old)) + t <- IO(test).onError(_ => ref.set(old)) _ <- ref.set(old) - } yield t).accepted + } yield t).unsafeRunSync() - def attributes(filename: String = "file.txt", size: Long = 12, id: UUID = uuid): FileAttributes = { - val uuidPathSegment = id.toString.take(8).mkString("/") - FileAttributes( - id, - s"file://$path/org/proj/$uuidPathSegment/$filename", - Uri.Path(s"org/proj/$uuidPathSegment/$filename"), - filename, - Some(`text/plain(UTF-8)`), - size, - digest, - Client - ) - } + def attributes( + filename: String = "file.txt", + size: Long = 12, + id: UUID = uuid, + projRef: ProjectRef = projectRef + ): FileAttributes = FileGen.attributes(filename, size, id, projRef, path) def entity(filename: String = "file.txt"): MessageEntity = Multipart diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FileGen.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FileGen.scala deleted file mode 100644 index c6b63b13cc..0000000000 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FileGen.scala +++ /dev/null @@ -1,56 +0,0 @@ -package ch.epfl.bluebrain.nexus.delta.plugins.storage.files - -import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes, FileState} -import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageType -import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri -import ch.epfl.bluebrain.nexus.delta.sdk.model.Tags -import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.{Anonymous, Subject} -import ch.epfl.bluebrain.nexus.delta.sourcing.model.{ProjectRef, ResourceRef} - -import java.time.Instant - -object FileGen { - - def state( - id: Iri, - project: ProjectRef, - storage: ResourceRef.Revision, - attributes: FileAttributes, - storageType: StorageType = StorageType.DiskStorage, - rev: Int = 1, - deprecated: Boolean = false, - tags: Tags = Tags.empty, - createdBy: Subject = Anonymous, - updatedBy: Subject = Anonymous - ): FileState = { - FileState( - id, - project, - storage, - storageType, - attributes, - tags, - rev, - deprecated, - Instant.EPOCH, - createdBy, - Instant.EPOCH, - updatedBy - ) - } - - def resourceFor( - id: Iri, - project: ProjectRef, - storage: ResourceRef.Revision, - attributes: FileAttributes, - storageType: StorageType = StorageType.DiskStorage, - rev: Int = 1, - deprecated: Boolean = false, - tags: Tags = Tags.empty, - createdBy: Subject = Anonymous, - updatedBy: Subject = Anonymous - ): FileResource = - state(id, project, storage, attributes, storageType, rev, deprecated, tags, createdBy, updatedBy).toResource - -} diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala index 9f6e2aa4bc..d9eebc1bae 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesSpec.scala @@ -8,6 +8,7 @@ import akka.testkit.TestKit import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.kernel.http.MediaTypeDetectorConfig import ch.epfl.bluebrain.nexus.delta.plugins.storage.RemoteContextResolutionFixture +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.generators.FileGen import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest.NotComputedDigest import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes.FileAttributesOrigin.Storage import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection._ @@ -26,7 +27,7 @@ import ch.epfl.bluebrain.nexus.delta.sdk.acls.model.AclAddress import ch.epfl.bluebrain.nexus.delta.sdk.auth.{AuthTokenProvider, Credentials} import ch.epfl.bluebrain.nexus.delta.sdk.directives.FileResponse import ch.epfl.bluebrain.nexus.delta.sdk.error.ServiceError.AuthorizationFailed -import ch.epfl.bluebrain.nexus.delta.sdk.http.{HttpClient, HttpClientConfig} +import ch.epfl.bluebrain.nexus.delta.sdk.http.HttpClient import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.{Caller, ServiceAccount} import ch.epfl.bluebrain.nexus.delta.sdk.implicits._ import ch.epfl.bluebrain.nexus.delta.sdk.model._ @@ -62,9 +63,8 @@ class FilesSpec(docker: RemoteStorageDocker) private val alice = User("Alice", realm) "The Files operations bundle" when { - implicit val hcc: HttpClientConfig = httpClientConfig implicit val typedSystem: typed.ActorSystem[Nothing] = system.toTyped - implicit val httpClient: HttpClient = HttpClient() + implicit val httpClient: HttpClient = HttpClient()(httpClientConfig, system) implicit val caller: Caller = Caller(bob, Set(bob, Group("mygroup", realm), Authenticated(realm))) implicit val authTokenProvider: AuthTokenProvider = AuthTokenProvider.anonymousForTest val remoteDiskStorageClient = new RemoteDiskStorageClient(httpClient, authTokenProvider, Credentials.Anonymous) @@ -631,13 +631,12 @@ class FilesSpec(docker: RemoteStorageDocker) assertion(id) } - def givenADeprecatedFile(assertion: FileId => Assertion): Assertion = { + def givenADeprecatedFile(assertion: FileId => Assertion): Assertion = givenAFile { id => files.deprecate(id, 1).accepted files.fetch(id).accepted.deprecated shouldEqual true assertion(id) } - } def assertRemainsDeprecated(id: FileId): Assertion = files.fetch(id).accepted.deprecated shouldEqual true diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesStmSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesStmSpec.scala index 0f6d1599b4..49b8d03841 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesStmSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/FilesStmSpec.scala @@ -2,6 +2,7 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.files import akka.http.scaladsl.model.{ContentTypes, Uri} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.Files.{evaluate, next} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.generators.FileGen import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest.{ComputedDigest, NotComputedDigest} import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes.FileAttributesOrigin.Client import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileCommand._ diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopySuite.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopySuite.scala new file mode 100644 index 0000000000..014ad46d98 --- /dev/null +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchCopySuite.scala @@ -0,0 +1,293 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch + +import cats.data.NonEmptyList +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch.BatchCopySuite._ +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.generators.FileGen +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.mocks._ +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes, FileDescription, FileId} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.CopyFileSource +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{FetchFileResource, FileFixtures, FileResource} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages._ +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.{DiskStorage, RemoteDiskStorage, S3Storage} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{Storage, StorageStatEntry, StorageType} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.CopyFileRejection.{DifferentStorageTypes, SourceFileTooLarge, TotalCopySizeTooLarge, UnsupportedOperation} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.{DiskCopyDetails, DiskStorageCopyFiles} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.RemoteDiskStorageCopyFiles +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.model.RemoteDiskCopyDetails +import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.nxv +import ch.epfl.bluebrain.nexus.delta.sdk.acls.model.AclAddress +import ch.epfl.bluebrain.nexus.delta.sdk.acls.{AclCheck, AclSimpleCheck} +import ch.epfl.bluebrain.nexus.delta.sdk.error.ServiceError.AuthorizationFailed +import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller +import ch.epfl.bluebrain.nexus.delta.sdk.model.{IdSegment, IdSegmentRef, Tags} +import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.User +import ch.epfl.bluebrain.nexus.delta.sourcing.model.{ProjectRef, ResourceRef} +import ch.epfl.bluebrain.nexus.testkit.Generators +import ch.epfl.bluebrain.nexus.testkit.mu.NexusSuite + +import java.util.UUID +import scala.collection.mutable.ListBuffer + +class BatchCopySuite extends NexusSuite with StorageFixtures with Generators with FileFixtures with FileGen { + + private val sourceFileDescUuid = UUID.randomUUID() + implicit private val fixedUUidF: UUIDF = UUIDF.fixed(sourceFileDescUuid) + + private val sourceProj = genProject() + private val sourceFileId = genFileId(sourceProj.ref) + private val source = CopyFileSource(sourceProj.ref, NonEmptyList.of(sourceFileId)) + private val storageStatEntry = StorageStatEntry(files = 10L, spaceUsed = 5L) + private val stubbedFileAttr = NonEmptyList.of(attributes(genString())) + + test("successfully perform disk copy") { + val events = ListBuffer.empty[Event] + val (sourceFileRes, sourceStorage) = genFileResourceAndStorage(sourceFileId, sourceProj.context, diskVal) + val (user, aclCheck) = userAuthorizedOnProjectStorage(sourceStorage.value) + + val batchCopy = mkBatchCopy( + fetchFile = stubbedFetchFile(sourceFileRes, events), + fetchStorage = stubbedFetchStorage(sourceStorage, events), + aclCheck = aclCheck, + stats = stubbedStorageStats(storageStatEntry, events), + diskCopy = stubbedDiskCopy(stubbedFileAttr, events) + ) + val destStorage: DiskStorage = genDiskStorage() + + batchCopy.copyFiles(source, destStorage)(caller(user)).map { obtained => + val obtainedEvents = events.toList + assertEquals(obtained, stubbedFileAttr) + sourceFileWasFetched(obtainedEvents, sourceFileId) + sourceStorageWasFetched(obtainedEvents, sourceFileRes.value.storage, sourceProj.ref) + destinationDiskStorageStatsWereFetched(obtainedEvents, destStorage) + diskCopyWasPerformed(obtainedEvents, destStorage, sourceFileRes.value.attributes) + } + } + + test("successfully perform remote disk copy") { + val events = ListBuffer.empty[Event] + val (sourceFileRes, sourceStorage) = genFileResourceAndStorage(sourceFileId, sourceProj.context, remoteVal) + val (user, aclCheck) = userAuthorizedOnProjectStorage(sourceStorage.value) + + val batchCopy = mkBatchCopy( + fetchFile = stubbedFetchFile(sourceFileRes, events), + fetchStorage = stubbedFetchStorage(sourceStorage, events), + aclCheck = aclCheck, + stats = stubbedStorageStats(storageStatEntry, events), + remoteCopy = stubbedRemoteCopy(stubbedFileAttr, events) + ) + val destStorage: RemoteDiskStorage = genRemoteStorage() + + batchCopy.copyFiles(source, destStorage)(caller(user)).map { obtained => + val obtainedEvents = events.toList + assertEquals(obtained, stubbedFileAttr) + sourceFileWasFetched(obtainedEvents, sourceFileId) + sourceStorageWasFetched(obtainedEvents, sourceFileRes.value.storage, sourceProj.ref) + destinationRemoteStorageStatsWereNotFetched(obtainedEvents) + remoteDiskCopyWasPerformed(obtainedEvents, destStorage, sourceFileRes.value.attributes) + } + } + + test("fail if destination storage is S3") { + val batchCopy = mkBatchCopy() + val (user, destStorage) = (genUser(), genS3Storage()) + val expectedError = UnsupportedOperation(StorageType.S3Storage) + batchCopy.copyFiles(source, destStorage)(caller(user)).interceptEquals(expectedError) + } + + test("fail if a source storage is different to destination storage") { + val events = ListBuffer.empty[Event] + val (sourceFileRes, sourceStorage) = genFileResourceAndStorage(sourceFileId, sourceProj.context, diskVal) + val (user, aclCheck) = userAuthorizedOnProjectStorage(sourceStorage.value) + + val batchCopy = mkBatchCopy( + fetchFile = stubbedFetchFile(sourceFileRes, events), + fetchStorage = stubbedFetchStorage(sourceStorage, events), + aclCheck = aclCheck + ) + val expectedError = DifferentStorageTypes(sourceStorage.id, StorageType.DiskStorage, StorageType.RemoteDiskStorage) + + batchCopy.copyFiles(source, genRemoteStorage())(caller(user)).interceptEquals(expectedError).map { _ => + val obtainedEvents = events.toList + sourceFileWasFetched(obtainedEvents, sourceFileId) + sourceStorageWasFetched(obtainedEvents, sourceFileRes.value.storage, sourceProj.ref) + } + } + + test("fail if user does not have read access on a source file's storage") { + val events = ListBuffer.empty[Event] + val (sourceFileRes, sourceStorage) = genFileResourceAndStorage(sourceFileId, sourceProj.context, diskVal) + val user = genUser() + val aclCheck = AclSimpleCheck((user, AclAddress.fromProject(sourceProj.ref), Set())).accepted + + val batchCopy = mkBatchCopy( + fetchFile = stubbedFetchFile(sourceFileRes, events), + fetchStorage = stubbedFetchStorage(sourceStorage, events), + aclCheck = aclCheck + ) + + batchCopy.copyFiles(source, genDiskStorage())(caller(user)).intercept[AuthorizationFailed].map { _ => + val obtainedEvents = events.toList + sourceFileWasFetched(obtainedEvents, sourceFileId) + sourceStorageWasFetched(obtainedEvents, sourceFileRes.value.storage, sourceProj.ref) + } + } + + test("fail if a single source file exceeds max size for destination storage") { + val events = ListBuffer.empty[Event] + val (sourceFileRes, sourceStorage) = genFileResourceAndStorage(sourceFileId, sourceProj.context, diskVal, 1000L) + val (user, aclCheck) = userAuthorizedOnProjectStorage(sourceStorage.value) + + val batchCopy = mkBatchCopy( + fetchFile = stubbedFetchFile(sourceFileRes, events), + fetchStorage = stubbedFetchStorage(sourceStorage, events), + aclCheck = aclCheck + ) + val destStorage = genDiskStorage() + val error = SourceFileTooLarge(destStorage.value.maxFileSize, destStorage.id) + + batchCopy.copyFiles(source, destStorage)(caller(user)).interceptEquals(error).map { _ => + val obtainedEvents = events.toList + sourceFileWasFetched(obtainedEvents, sourceFileId) + sourceStorageWasFetched(obtainedEvents, sourceFileRes.value.storage, sourceProj.ref) + } + } + + test("fail if total size of source files is too large for destination disk storage") { + val events = ListBuffer.empty[Event] + val fileSize = 6L + val capacity = 10L + val statEntry = StorageStatEntry(files = 10L, spaceUsed = 1L) + val spaceLeft = capacity - statEntry.spaceUsed + val (sourceFileRes, sourceStorage) = genFileResourceAndStorage(sourceFileId, sourceProj.context, diskVal, fileSize) + val (user, aclCheck) = userAuthorizedOnProjectStorage(sourceStorage.value) + + val batchCopy = mkBatchCopy( + fetchFile = stubbedFetchFile(sourceFileRes, events), + fetchStorage = stubbedFetchStorage(sourceStorage, events), + aclCheck = aclCheck, + stats = stubbedStorageStats(statEntry, events) + ) + + val destStorage = genDiskStorageWithCapacity(capacity) + val error = TotalCopySizeTooLarge(fileSize * 2, spaceLeft, destStorage.id) + val twoFileSource = CopyFileSource(sourceProj.ref, NonEmptyList.of(sourceFileId, sourceFileId)) + + batchCopy.copyFiles(twoFileSource, destStorage)(caller(user)).interceptEquals(error).map { _ => + val obtainedEvents = events.toList + sourceFileWasFetched(obtainedEvents, sourceFileId) + sourceStorageWasFetched(obtainedEvents, sourceFileRes.value.storage, sourceProj.ref) + destinationDiskStorageStatsWereFetched(obtainedEvents, destStorage) + } + } + + private def mkBatchCopy( + fetchFile: FetchFileResource = FetchFileResourceMock.unimplemented, + fetchStorage: FetchStorage = FetchStorageMock.unimplemented, + aclCheck: AclCheck = AclSimpleCheck().accepted, + stats: StoragesStatistics = StoragesStatisticsMock.unimplemented, + diskCopy: DiskStorageCopyFiles = DiskCopyMock.unimplemented, + remoteCopy: RemoteDiskStorageCopyFiles = RemoteCopyMock.unimplemented + ): BatchCopy = BatchCopy.mk(fetchFile, fetchStorage, aclCheck, stats, diskCopy, remoteCopy) + + private def userAuthorizedOnProjectStorage(storage: Storage): (User, AclCheck) = { + val user = genUser() + val permissions = Set(storage.storageValue.readPermission) + (user, AclSimpleCheck((user, AclAddress.fromProject(storage.project), permissions)).accepted) + } + + private def sourceFileWasFetched(events: List[Event], id: FileId) = { + val obtained = events.collectFirst { case f: FetchFileCalled => f } + assertEquals(obtained, Some(FetchFileCalled(id))) + } + + private def sourceStorageWasFetched(events: List[Event], storageRef: ResourceRef.Revision, proj: ProjectRef) = { + val obtained = events.collectFirst { case f: FetchStorageCalled => f } + assertEquals(obtained, Some(FetchStorageCalled(IdSegmentRef(storageRef), proj))) + } + + private def destinationDiskStorageStatsWereFetched(events: List[Event], storage: DiskStorage) = { + val obtained = events.collectFirst { case f: StoragesStatsCalled => f } + assertEquals(obtained, Some(StoragesStatsCalled(storage.id, storage.project))) + } + + private def destinationRemoteStorageStatsWereNotFetched(events: List[Event]) = { + val obtained = events.collectFirst { case f: StoragesStatsCalled => f } + assertEquals(obtained, None) + } + + private def diskCopyWasPerformed(events: List[Event], storage: DiskStorage, sourceAttr: FileAttributes) = { + val expectedDesc = FileDescription(sourceFileDescUuid, sourceAttr.filename, sourceAttr.mediaType) + val expectedDiskCopyDetails = DiskCopyDetails(storage, expectedDesc, sourceAttr) + val obtained = events.collectFirst { case f: DiskCopyCalled => f } + assertEquals(obtained, Some(DiskCopyCalled(storage, NonEmptyList.of(expectedDiskCopyDetails)))) + } + + private def remoteDiskCopyWasPerformed( + events: List[Event], + storage: RemoteDiskStorage, + sourceAttr: FileAttributes + ) = { + val expectedDesc = FileDescription(sourceFileDescUuid, sourceAttr.filename, sourceAttr.mediaType) + val expectedCopyDetails = RemoteDiskCopyDetails(storage, expectedDesc, storage.value.folder, sourceAttr) + val obtained = events.collectFirst { case f: RemoteCopyCalled => f } + assertEquals(obtained, Some(RemoteCopyCalled(storage, NonEmptyList.of(expectedCopyDetails)))) + } + + private def caller(user: User): Caller = Caller(user, Set(user)) + + private def genDiskStorageWithCapacity(capacity: Long) = { + val limitedDiskVal = diskVal.copy(capacity = Some(capacity)) + DiskStorage(nxv + genString(), genProject().ref, limitedDiskVal, Tags.empty, json"""{"disk": "value"}""") + } + + private def genDiskStorage() = genDiskStorageWithCapacity(1000L) + + private def genRemoteStorage() = + RemoteDiskStorage(nxv + genString(), genProject().ref, remoteVal, Tags.empty, json"""{"disk": "value"}""") + + private def genS3Storage() = + S3Storage(nxv + genString(), genProject().ref, s3Val, Tags.empty, json"""{"disk": "value"}""") +} + +object BatchCopySuite { + sealed trait Event + final case class FetchFileCalled(id: FileId) extends Event + final case class FetchStorageCalled(id: IdSegmentRef, project: ProjectRef) extends Event + final case class StoragesStatsCalled(idSegment: IdSegment, project: ProjectRef) extends Event + final case class DiskCopyCalled(destStorage: Storage.DiskStorage, details: NonEmptyList[DiskCopyDetails]) + extends Event + final case class RemoteCopyCalled( + destStorage: Storage.RemoteDiskStorage, + copyDetails: NonEmptyList[RemoteDiskCopyDetails] + ) extends Event + + private def stubbedStorageStats(storageStatEntry: StorageStatEntry, events: ListBuffer[Event]) = + StoragesStatisticsMock.withMockedGet((id, proj) => + addEventAndReturn(events, StoragesStatsCalled(id, proj), storageStatEntry) + ) + + private def stubbedFetchFile(sourceFileRes: FileResource, events: ListBuffer[Event]) = + FetchFileResourceMock.withMockedFetch(id => addEventAndReturn(events, FetchFileCalled(id), sourceFileRes)) + + private def stubbedFetchStorage(storage: StorageResource, events: ListBuffer[Event]) = + FetchStorageMock.withMockedFetch((id, proj) => addEventAndReturn(events, FetchStorageCalled(id, proj), storage)) + + private def stubbedRemoteCopy(stubbedRemoteFileAttr: NonEmptyList[FileAttributes], events: ListBuffer[Event]) = + RemoteCopyMock.withMockedCopy((storage, details) => + addEventAndReturn(events, RemoteCopyCalled(storage, details), stubbedRemoteFileAttr) + ) + + private def stubbedDiskCopy(stubbedDiskFileAttr: NonEmptyList[FileAttributes], events: ListBuffer[Event]) = + DiskCopyMock.withMockedCopy((storage, details) => + addEventAndReturn(events, DiskCopyCalled(storage, details), stubbedDiskFileAttr) + ) + + private def addEventAndReturn[A](events: ListBuffer[Event], event: Event, a: A): IO[A] = + addEventAndReturnIO(events, event, IO.pure(a)) + + private def addEventAndReturnIO[A](events: ListBuffer[Event], event: Event, io: IO[A]): IO[A] = + IO(events.addOne(event)) >> io + +} diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFilesSuite.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFilesSuite.scala new file mode 100644 index 0000000000..30d1ee6b82 --- /dev/null +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/batch/BatchFilesSuite.scala @@ -0,0 +1,129 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch + +import cats.data.NonEmptyList +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.kernel.utils.UUIDF +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch.BatchFilesSuite._ +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.generators.FileGen +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.mocks.BatchCopyMock +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileCommand.CreateFile +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection.CopyRejection +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes, FileCommand, FileRejection} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.CopyFileSource +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{FetchFileStorage, FileFixtures, FileResource} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StorageFixtures +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.CopyFileRejection.TotalCopySizeTooLarge +import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller +import ch.epfl.bluebrain.nexus.delta.sdk.model.IdSegment +import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.{Project, ProjectContext} +import ch.epfl.bluebrain.nexus.delta.sdk.projects.{FetchContext, FetchContextDummy} +import ch.epfl.bluebrain.nexus.delta.sourcing.model.{ProjectRef, ResourceRef} +import ch.epfl.bluebrain.nexus.testkit.Generators +import ch.epfl.bluebrain.nexus.testkit.mu.NexusSuite + +import java.util.UUID +import scala.collection.mutable.ListBuffer + +class BatchFilesSuite extends NexusSuite with StorageFixtures with Generators with FileFixtures with FileGen { + + private val destProj: Project = genProject() + private val (destStorageRef, destStorage) = (genRevision(), genStorage(destProj.ref, diskVal)) + private val destFileUUId = UUID.randomUUID() // Not testing UUID generation, same for all of them + private val destination = genCopyFileDestination(destProj.ref, destStorage.storage) + + test("batch copying should fetch storage, perform copy and evaluate create file commands") { + val events = ListBuffer.empty[Event] + val fetchFileStorage = mockFetchFileStorage(destStorageRef, destStorage.storage, events) + val stubbedDestAttributes = genAttributes() + val batchCopy = BatchCopyMock.withStubbedCopyFiles(events, stubbedDestAttributes) + + val batchFiles: BatchFiles = mkBatchFiles(events, destProj, destFileUUId, fetchFileStorage, batchCopy) + implicit val c: Caller = Caller(genUser(), Set()) + val source = genCopyFileSource() + + batchFiles.copyFiles(source, destination).map { obtained => + val expectedCommands = createCommandsFromFileAttributes(stubbedDestAttributes) + val expectedResources = expectedCommands.map(genFileResourceFromCmd) + val expectedCommandCalls = expectedCommands.toList.map(FileCommandEvaluated) + val expectedEvents = activeStorageFetchedAndBatchCopyCalled(source) ++ expectedCommandCalls + + assertEquals(obtained, expectedResources) + assertEquals(events.toList, expectedEvents) + } + } + + test("copy rejections should be mapped to a file rejection") { + val events = ListBuffer.empty[Event] + val fetchFileStorage = mockFetchFileStorage(destStorageRef, destStorage.storage, events) + val error = TotalCopySizeTooLarge(1L, 2L, genIri()) + val batchCopy = BatchCopyMock.withError(error, events) + + val batchFiles: BatchFiles = mkBatchFiles(events, destProj, UUID.randomUUID(), fetchFileStorage, batchCopy) + implicit val c: Caller = Caller(genUser(), Set()) + val source = genCopyFileSource() + val expectedError = CopyRejection(source.project, destProj.ref, destStorage.id, error) + + batchFiles.copyFiles(source, destination).interceptEquals(expectedError).accepted + + assertEquals(events.toList, activeStorageFetchedAndBatchCopyCalled(source)) + } + + def mockFetchFileStorage( + storageRef: ResourceRef.Revision, + storage: Storage, + events: ListBuffer[Event] + ): FetchFileStorage = new FetchFileStorage { + override def fetchAndValidateActiveStorage(storageIdOpt: Option[IdSegment], ref: ProjectRef, pc: ProjectContext)( + implicit caller: Caller + ): IO[(ResourceRef.Revision, Storage)] = + IO(events.addOne(ActiveStorageFetched(storageIdOpt, ref, pc, caller))).as(storageRef -> storage) + } + + def mkBatchFiles( + events: ListBuffer[Event], + proj: Project, + fixedUuid: UUID, + fetchFileStorage: FetchFileStorage, + batchCopy: BatchCopy + ): BatchFiles = { + implicit val uuidF: UUIDF = UUIDF.fixed(fixedUuid) + val evalFileCmd: CreateFile => IO[FileResource] = cmd => + IO(events.addOne(FileCommandEvaluated(cmd))).as(genFileResourceFromCmd(cmd)) + val fetchContext: FetchContext[FileRejection] = + FetchContextDummy(Map(proj.ref -> proj.context)).mapRejection(FileRejection.ProjectContextRejection) + BatchFiles.mk(fetchFileStorage, fetchContext, evalFileCmd, batchCopy) + } + + def activeStorageFetchedAndBatchCopyCalled(source: CopyFileSource)(implicit c: Caller): List[Event] = { + val expectedActiveStorageFetched = ActiveStorageFetched(destination.storage, destProj.ref, destProj.context, c) + val expectedBatchCopyCalled = BatchCopyCalled(source, destStorage.storage, c) + List(expectedActiveStorageFetched, expectedBatchCopyCalled) + } + + def createCommandsFromFileAttributes(stubbedDestAttributes: NonEmptyList[FileAttributes])(implicit + c: Caller + ): NonEmptyList[CreateFile] = stubbedDestAttributes.map( + CreateFile( + destProj.base.iri / destFileUUId.toString, + destProj.ref, + destStorageRef, + destStorage.value.tpe, + _, + c.subject, + destination.tag + ) + ) +} + +object BatchFilesSuite { + sealed trait Event + final case class ActiveStorageFetched( + storageIdOpt: Option[IdSegment], + ref: ProjectRef, + pc: ProjectContext, + caller: Caller + ) extends Event + final case class BatchCopyCalled(source: CopyFileSource, destStorage: Storage, caller: Caller) extends Event + final case class FileCommandEvaluated(cmd: FileCommand) extends Event +} diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/generators/FileGen.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/generators/FileGen.scala new file mode 100644 index 0000000000..35ff172471 --- /dev/null +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/generators/FileGen.scala @@ -0,0 +1,177 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.generators + +import akka.http.scaladsl.model.ContentTypes.`text/plain(UTF-8)` +import akka.http.scaladsl.model.Uri +import cats.data.NonEmptyList +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest.ComputedDigest +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes.FileAttributesOrigin.Client +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileCommand.CreateFile +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{CopyFileDestination, FileAttributes, FileId, FileState} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.CopyFileSource +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{schemas, FileFixtures, FileResource} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model._ +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{StorageGen, StorageResource} +import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri +import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.nxv +import ch.epfl.bluebrain.nexus.delta.sdk.generators.ProjectGen +import ch.epfl.bluebrain.nexus.delta.sdk.model.{IdSegment, Tags} +import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.{ApiMappings, Project, ProjectContext} +import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.{Anonymous, Subject, User} +import ch.epfl.bluebrain.nexus.delta.sourcing.model.Tag.UserTag +import ch.epfl.bluebrain.nexus.delta.sourcing.model.{Label, ProjectRef, ResourceRef} +import ch.epfl.bluebrain.nexus.testkit.Generators + +import java.nio.file.{Files => JavaFiles} +import java.time.Instant +import java.util.UUID +import scala.util.Random + +trait FileGen { self: Generators with FileFixtures => + def genProjectRef(): ProjectRef = ProjectRef.unsafe(genString(), genString()) + + def genProject(): Project = { + val projRef = genProjectRef() + val apiMappings = ApiMappings("file" -> schemas.files) + ProjectGen.project(projRef.project.value, projRef.organization.value, base = nxv.base, mappings = apiMappings) + } + + def genUser(realmLabel: Label): User = User(genString(), realmLabel) + def genUser(): User = User(genString(), Label.unsafe(genString())) + + def genFilesIdsInProject(projRef: ProjectRef): NonEmptyList[FileId] = + NonEmptyList.of(genFileId(projRef), genFileId(projRef)) + + def genFileId(projRef: ProjectRef) = FileId(genString(), projRef) + + def genFileIdWithRev(projRef: ProjectRef): FileId = FileId(genString(), 4, projRef) + + def genFileIdWithTag(projRef: ProjectRef): FileId = FileId(genString(), UserTag.unsafe(genString()), projRef) + + def genAttributes(): NonEmptyList[FileAttributes] = { + val proj = genProject() + genFilesIdsInProject(proj.ref).map(genFileResource(_, proj.context)).map(_.value.attributes) + } + + def genCopyFileSource(): CopyFileSource = genCopyFileSource(genProjectRef()) + def genCopyFileSource(proj: ProjectRef) = CopyFileSource(proj, genFilesIdsInProject(proj)) + def genCopyFileDestination(proj: ProjectRef, storage: Storage): CopyFileDestination = + CopyFileDestination(proj, genOption(IdSegment(storage.id.toString)), genOption(genUserTag)) + def genUserTag: UserTag = UserTag.unsafe(genString()) + def genOption[A](genA: => A): Option[A] = if (Random.nextInt(2) % 2 == 0) Some(genA) else None + + def genFileResource(fileId: FileId, context: ProjectContext): FileResource = + genFileResourceWithStorage(fileId, context, genRevision(), 1L) + + def genFileResourceWithStorage( + fileId: FileId, + context: ProjectContext, + storageRef: ResourceRef.Revision, + fileSize: Long + ): FileResource = + genFileResourceWithIri( + fileId.id.value.toIri(context.apiMappings, context.base).getOrElse(throw new Exception(s"Bad file $fileId")), + fileId.project, + storageRef, + attributes(genString(), size = fileSize) + ) + + def genFileResourceAndStorage( + fileId: FileId, + context: ProjectContext, + storageVal: StorageValue, + fileSize: Long = 1L + ): (FileResource, StorageResource) = { + val storageRes = StorageGen.resourceFor(genIri(), fileId.project, storageVal) + val storageRef = ResourceRef.Revision(storageRes.id, storageRes.id, storageRes.rev) + (genFileResourceWithStorage(fileId, context, storageRef, fileSize), storageRes) + } + + def genFileResourceWithIri( + iri: Iri, + projRef: ProjectRef, + storageRef: ResourceRef.Revision, + attr: FileAttributes + ): FileResource = + FileGen.resourceFor(iri, projRef, storageRef, attr) + + def genFileResourceFromCmd(cmd: CreateFile): FileResource = + genFileResourceWithIri(cmd.id, cmd.project, cmd.storage, cmd.attributes) + def genIri(): Iri = Iri.unsafe(genString()) + def genStorage(proj: ProjectRef, storageValue: StorageValue): StorageState = + StorageGen.storageState(genIri(), proj, storageValue) + + def genRevision(): ResourceRef.Revision = + ResourceRef.Revision(genIri(), genPosInt()) + def genPosInt(): Int = Random.nextInt(Int.MaxValue) +} + +object FileGen { + + def state( + id: Iri, + project: ProjectRef, + storage: ResourceRef.Revision, + attributes: FileAttributes, + storageType: StorageType = StorageType.DiskStorage, + rev: Int = 1, + deprecated: Boolean = false, + tags: Tags = Tags.empty, + createdBy: Subject = Anonymous, + updatedBy: Subject = Anonymous + ): FileState = { + FileState( + id, + project, + storage, + storageType, + attributes, + tags, + rev, + deprecated, + Instant.EPOCH, + createdBy, + Instant.EPOCH, + updatedBy + ) + } + + def resourceFor( + id: Iri, + project: ProjectRef, + storage: ResourceRef.Revision, + attributes: FileAttributes, + storageType: StorageType = StorageType.DiskStorage, + rev: Int = 1, + deprecated: Boolean = false, + tags: Tags = Tags.empty, + createdBy: Subject = Anonymous, + updatedBy: Subject = Anonymous + ): FileResource = + state(id, project, storage, attributes, storageType, rev, deprecated, tags, createdBy, updatedBy).toResource + + def mkTempDir(prefix: String) = + AbsolutePath(JavaFiles.createTempDirectory(prefix)).fold(e => throw new Exception(e), identity) + + private val digest = + ComputedDigest(DigestAlgorithm.default, "e0ac3601005dfa1864f5392aabaf7d898b1b5bab854f1acb4491bcd806b76b0c") + + def attributes( + filename: String, + size: Long, + id: UUID, + projRef: ProjectRef, + path: AbsolutePath + ): FileAttributes = { + val uuidPathSegment = id.toString.take(8).mkString("/") + FileAttributes( + id, + s"file://$path/${projRef.toString}/$uuidPathSegment/$filename", + Uri.Path(s"${projRef.toString}/$uuidPathSegment/$filename"), + filename, + Some(`text/plain(UTF-8)`), + size, + digest, + Client + ) + } +} diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/BatchCopyMock.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/BatchCopyMock.scala new file mode 100644 index 0000000000..eb9d398892 --- /dev/null +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/BatchCopyMock.scala @@ -0,0 +1,36 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.mocks + +import cats.data.NonEmptyList +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch.BatchFilesSuite.{BatchCopyCalled, Event} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch.BatchCopy +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.CopyFileSource +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.CopyFileRejection +import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller + +import scala.collection.mutable.ListBuffer + +object BatchCopyMock { + + def withError(e: CopyFileRejection, events: ListBuffer[Event]): BatchCopy = + withMockedCopyFiles((source, destStorage) => + caller => IO(events.addOne(BatchCopyCalled(source, destStorage, caller))) >> IO.raiseError(e) + ) + + def withStubbedCopyFiles(events: ListBuffer[Event], stubbedAttr: NonEmptyList[FileAttributes]): BatchCopy = + withMockedCopyFiles((source, destStorage) => + caller => IO(events.addOne(BatchCopyCalled(source, destStorage, caller))).as(stubbedAttr) + ) + + def withMockedCopyFiles( + copyFilesMock: (CopyFileSource, Storage) => Caller => IO[NonEmptyList[FileAttributes]] + ): BatchCopy = + new BatchCopy { + override def copyFiles(source: CopyFileSource, destStorage: Storage)(implicit + c: Caller + ): IO[NonEmptyList[FileAttributes]] = copyFilesMock(source, destStorage)(c) + } + +} diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/BatchFilesMock.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/BatchFilesMock.scala new file mode 100644 index 0000000000..f487252f3e --- /dev/null +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/BatchFilesMock.scala @@ -0,0 +1,60 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.mocks + +import cats.data.NonEmptyList +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.FileResource +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch.BatchFiles +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{CopyFileDestination, FileId} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes.CopyFileSource +import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller +import ch.epfl.bluebrain.nexus.delta.sdk.model.IdSegment +import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.User +import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef +import ch.epfl.bluebrain.nexus.delta.sourcing.model.Tag.UserTag + +import scala.collection.mutable.ListBuffer + +object BatchFilesMock { + + def unimplemented: BatchFiles = withMockedCopyFiles((_, _) => _ => IO(???)) + + def withStubbedCopyFiles( + stubbed: NonEmptyList[FileResource], + events: ListBuffer[BatchFilesCopyFilesCalled] + ): BatchFiles = + withMockedCopyFiles((source, dest) => + c => IO(events.addOne(BatchFilesCopyFilesCalled(source, dest, c))).as(stubbed) + ) + + def withError(e: Throwable, events: ListBuffer[BatchFilesCopyFilesCalled]): BatchFiles = + withMockedCopyFiles((source, dest) => + c => IO(events.addOne(BatchFilesCopyFilesCalled(source, dest, c))) >> IO.raiseError(e) + ) + + def withMockedCopyFiles( + copyFilesMock: (CopyFileSource, CopyFileDestination) => Caller => IO[NonEmptyList[FileResource]] + ): BatchFiles = new BatchFiles { + override def copyFiles(source: CopyFileSource, dest: CopyFileDestination)(implicit + c: Caller + ): IO[NonEmptyList[FileResource]] = + copyFilesMock(source, dest)(c) + } + + final case class BatchFilesCopyFilesCalled(source: CopyFileSource, dest: CopyFileDestination, caller: Caller) + + object BatchFilesCopyFilesCalled { + def fromTestData( + destProj: ProjectRef, + sourceProj: ProjectRef, + sourceFiles: NonEmptyList[FileId], + user: User, + destStorage: Option[IdSegment] = None, + destTag: Option[UserTag] = None + ): BatchFilesCopyFilesCalled = { + val expectedCopyFileSource = CopyFileSource(sourceProj, sourceFiles) + val expectedCopyFileDestination = CopyFileDestination(destProj, destStorage, destTag) + BatchFilesCopyFilesCalled(expectedCopyFileSource, expectedCopyFileDestination, Caller(user, Set(user))) + } + } + +} diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/DiskCopyMock.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/DiskCopyMock.scala new file mode 100644 index 0000000000..3ad1fbe023 --- /dev/null +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/DiskCopyMock.scala @@ -0,0 +1,17 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.mocks + +import cats.data.NonEmptyList +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.DiskStorage +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.{DiskCopyDetails, DiskStorageCopyFiles} + +object DiskCopyMock { + + def unimplemented: DiskStorageCopyFiles = withMockedCopy((_, _) => IO(???)) + + def withMockedCopy( + copyMock: (DiskStorage, NonEmptyList[DiskCopyDetails]) => IO[NonEmptyList[FileAttributes]] + ): DiskStorageCopyFiles = copyMock(_, _) + +} diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/FetchFileResourceMock.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/FetchFileResourceMock.scala new file mode 100644 index 0000000000..3a4f2d93d9 --- /dev/null +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/FetchFileResourceMock.scala @@ -0,0 +1,13 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.mocks + +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileId +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{FetchFileResource, FileResource} + +object FetchFileResourceMock { + + def unimplemented: FetchFileResource = withMockedFetch(_ => IO(???)) + + def withMockedFetch(fetchMock: FileId => IO[FileResource]): FetchFileResource = fetchMock(_) + +} diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/FetchStorageMock.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/FetchStorageMock.scala new file mode 100644 index 0000000000..cd45218d77 --- /dev/null +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/FetchStorageMock.scala @@ -0,0 +1,17 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.mocks + +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{FetchStorage, StorageResource} +import ch.epfl.bluebrain.nexus.delta.sdk.model.IdSegmentRef +import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef + +object FetchStorageMock { + + def unimplemented: FetchStorage = withMockedFetch((_, _) => IO(???)) + + def withMockedFetch(fetchMock: (IdSegmentRef, ProjectRef) => IO[StorageResource]): FetchStorage = new FetchStorage { + override def fetch(id: IdSegmentRef, project: ProjectRef): IO[StorageResource] = fetchMock(id, project) + override def fetchDefault(project: ProjectRef): IO[StorageResource] = ??? + } + +} diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/RemoteCopyMock.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/RemoteCopyMock.scala new file mode 100644 index 0000000000..ed80532a91 --- /dev/null +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/RemoteCopyMock.scala @@ -0,0 +1,18 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.mocks + +import cats.data.NonEmptyList +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileAttributes +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.RemoteDiskStorage +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.RemoteDiskStorageCopyFiles +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.model.RemoteDiskCopyDetails + +object RemoteCopyMock { + + def unimplemented: RemoteDiskStorageCopyFiles = withMockedCopy((_, _) => IO(???)) + + def withMockedCopy( + copyMock: (RemoteDiskStorage, NonEmptyList[RemoteDiskCopyDetails]) => IO[NonEmptyList[FileAttributes]] + ): RemoteDiskStorageCopyFiles = copyMock(_, _) + +} diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/StoragesStatisticsMock.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/StoragesStatisticsMock.scala new file mode 100644 index 0000000000..1b2c918233 --- /dev/null +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/mocks/StoragesStatisticsMock.scala @@ -0,0 +1,15 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.mocks + +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StoragesStatistics +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageStatEntry +import ch.epfl.bluebrain.nexus.delta.sdk.model.IdSegment +import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef + +object StoragesStatisticsMock { + + def unimplemented: StoragesStatistics = withMockedGet((_, _) => IO(???)) + + def withMockedGet(getMock: (IdSegment, ProjectRef) => IO[StorageStatEntry]): StoragesStatistics = + (idSegment: IdSegment, project: ProjectRef) => getMock(idSegment, project) +} diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesRoutesSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesRoutesSpec.scala new file mode 100644 index 0000000000..00aef9481b --- /dev/null +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/BatchFilesRoutesSpec.scala @@ -0,0 +1,271 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.storage.files.routes + +import akka.http.scaladsl.model.headers.OAuth2BearerToken +import akka.http.scaladsl.model.{StatusCode, StatusCodes} +import akka.http.scaladsl.server.Route +import cats.data.NonEmptyList +import ch.epfl.bluebrain.nexus.delta.kernel.utils.ClassUtils +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.batch.BatchFiles +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.generators.FileGen +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.mocks.BatchFilesMock +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.mocks.BatchFilesMock.BatchFilesCopyFilesCalled +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.FileRejection.{CopyRejection, FileNotFound, WrappedStorageRejection} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileId, FileRejection} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.{contexts => fileContexts, FileFixtures, FileResource} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StorageFixtures +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageRejection.{DifferentStorageType, StorageNotFound} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageType +import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.CopyFileRejection.{SourceFileTooLarge, TotalCopySizeTooLarge, UnsupportedOperation} +import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.{ContextValue, RemoteContextResolution} +import ch.epfl.bluebrain.nexus.delta.sdk.IndexingAction +import ch.epfl.bluebrain.nexus.delta.sdk.acls.model.AclAddress +import ch.epfl.bluebrain.nexus.delta.sdk.acls.{AclCheck, AclSimpleCheck} +import ch.epfl.bluebrain.nexus.delta.sdk.directives.DeltaSchemeDirectives +import ch.epfl.bluebrain.nexus.delta.sdk.identities.IdentitiesDummy +import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.Caller +import ch.epfl.bluebrain.nexus.delta.sdk.model.{IdSegment, IdSegmentRef} +import ch.epfl.bluebrain.nexus.delta.sdk.permissions.model.Permission +import ch.epfl.bluebrain.nexus.delta.sdk.projects.FetchContextDummy +import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.Project +import ch.epfl.bluebrain.nexus.delta.sdk.utils.BaseRouteSpec +import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.User +import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef +import ch.epfl.bluebrain.nexus.delta.sourcing.model.Tag.UserTag +import io.circe.Json +import io.circe.syntax.KeyOps +import org.scalatest.Assertion + +import scala.collection.mutable.ListBuffer + +class BatchFilesRoutesSpec extends BaseRouteSpec with StorageFixtures with FileFixtures with FileGen { + + implicit override def rcr: RemoteContextResolution = + RemoteContextResolution.fixedIO( + fileContexts.files -> ContextValue.fromFile("contexts/files.json"), + Vocabulary.contexts.metadata -> ContextValue.fromFile("contexts/metadata.json"), + Vocabulary.contexts.bulkOperation -> ContextValue.fromFile("contexts/bulk-operation.json"), + Vocabulary.contexts.error -> ContextValue.fromFile("contexts/error.json") + ) + + "Batch copying files between projects" should { + + "succeed for source files looked up by latest" in { + val sourceProj = genProject() + val sourceFileIds = genFilesIdsInProject(sourceProj.ref) + testBulkCopySucceedsForStubbedFiles(sourceProj, sourceFileIds) + } + + "succeed for source files looked up by tag" in { + val sourceProj = genProject() + val sourceFileIds = NonEmptyList.of(genFileIdWithTag(sourceProj.ref), genFileIdWithTag(sourceProj.ref)) + testBulkCopySucceedsForStubbedFiles(sourceProj, sourceFileIds) + } + + "succeed for source files looked up by rev" in { + val sourceProj = genProject() + val sourceFileIds = NonEmptyList.of(genFileIdWithRev(sourceProj.ref), genFileIdWithRev(sourceProj.ref)) + testBulkCopySucceedsForStubbedFiles(sourceProj, sourceFileIds) + } + + "succeed with a specific destination storage" in { + val sourceProj = genProject() + val sourceFileIds = genFilesIdsInProject(sourceProj.ref) + val destStorageId = IdSegment(genString()) + testBulkCopySucceedsForStubbedFiles(sourceProj, sourceFileIds, destStorageId = Some(destStorageId)) + } + + "succeed with a user tag applied to destination files" in { + val sourceProj = genProject() + val sourceFileIds = genFilesIdsInProject(sourceProj.ref) + val destTag = UserTag.unsafe(genString()) + testBulkCopySucceedsForStubbedFiles(sourceProj, sourceFileIds, destTag = Some(destTag)) + } + + "return 403 for a user without read permission on the source project" in { + val (sourceProj, destProj, user) = (genProject(), genProject(), genUser(realm)) + val sourceFileIds = genFilesIdsInProject(sourceProj.ref) + + val route = mkRoute(BatchFilesMock.unimplemented, sourceProj, user, permissions = Set()) + val payload = BatchFilesRoutesSpec.mkBulkCopyPayload(sourceProj.ref, sourceFileIds) + + callBulkCopyEndpoint(route, destProj.ref, payload, user) { + response.shouldBeForbidden + } + } + + "return 400 if tag and rev are present simultaneously for a source file" in { + val (sourceProj, destProj, user) = (genProject(), genProject(), genUser(realm)) + + val route = mkRoute(BatchFilesMock.unimplemented, sourceProj, user, permissions = Set()) + val invalidFilePayload = BatchFilesRoutesSpec.mkSourceFilePayload(genString(), Some(3), Some(genString())) + val payload = Json.obj("sourceProjectRef" := sourceProj.ref, "files" := List(invalidFilePayload)) + + callBulkCopyEndpoint(route, destProj.ref, payload, user) { + response.status shouldBe StatusCodes.BadRequest + } + } + + "return 400 for copy errors raised by batch file logic" in { + val unsupportedStorageType = UnsupportedOperation(StorageType.S3Storage) + val fileTooLarge = SourceFileTooLarge(12, genIri()) + val totalSizeTooLarge = TotalCopySizeTooLarge(1L, 2L, genIri()) + + val errors = List(unsupportedStorageType, fileTooLarge, totalSizeTooLarge) + .map(CopyRejection(genProjectRef(), genProjectRef(), genIri(), _)) + + forAll(errors) { error => + val (sourceProj, destProj, user) = (genProject(), genProject(), genUser(realm)) + val sourceFileIds = genFilesIdsInProject(sourceProj.ref) + val events = ListBuffer.empty[BatchFilesCopyFilesCalled] + val batchFiles = BatchFilesMock.withError(error, events) + + val route = mkRoute(batchFiles, sourceProj, user, permissions = Set(files.permissions.read)) + val payload = BatchFilesRoutesSpec.mkBulkCopyPayload(sourceProj.ref, sourceFileIds) + + callBulkCopyEndpoint(route, destProj.ref, payload, user) { + response.status shouldBe StatusCodes.BadRequest + response.asJson shouldBe errorJson(error, Some(ClassUtils.simpleName(error.rejection)), error.loggedDetails) + } + } + } + + "map other file rejections to the correct response" in { + val storageNotFound = WrappedStorageRejection(StorageNotFound(genIri(), genProjectRef())) + val differentStorageType = + WrappedStorageRejection(DifferentStorageType(genIri(), StorageType.DiskStorage, StorageType.RemoteDiskStorage)) + val fileNotFound = FileNotFound(genIri(), genProjectRef()) + + val fileRejections: List[(FileRejection, StatusCode, Json)] = List( + (storageNotFound, StatusCodes.NotFound, errorJson(storageNotFound, Some("ResourceNotFound"))), + (fileNotFound, StatusCodes.NotFound, errorJson(fileNotFound, Some("ResourceNotFound"))), + (differentStorageType, StatusCodes.BadRequest, errorJson(differentStorageType, Some("DifferentStorageType"))) + ) + + forAll(fileRejections) { case (error, expectedStatus, expectedJson) => + val (sourceProj, destProj, user) = (genProject(), genProject(), genUser(realm)) + val sourceFileIds = genFilesIdsInProject(sourceProj.ref) + val events = ListBuffer.empty[BatchFilesCopyFilesCalled] + val batchFiles = BatchFilesMock.withError(error, events) + + val route = mkRoute(batchFiles, sourceProj, user, permissions = Set(files.permissions.read)) + val payload = BatchFilesRoutesSpec.mkBulkCopyPayload(sourceProj.ref, sourceFileIds) + + callBulkCopyEndpoint(route, destProj.ref, payload, user) { + response.status shouldBe expectedStatus + response.asJson shouldBe expectedJson + } + } + } + } + + def errorJson(t: Throwable, specificType: Option[String] = None, details: Option[String] = None): Json = { + val detailsObj = details.fold(Json.obj())(d => Json.obj("details" := d)) + detailsObj.deepMerge( + Json.obj( + "@context" := Vocabulary.contexts.error.toString, + "@type" := specificType.getOrElse(ClassUtils.simpleName(t)), + "reason" := t.getMessage + ) + ) + } + + def mkRoute( + batchFiles: BatchFiles, + proj: Project, + user: User, + permissions: Set[Permission] + ): Route = { + val aclCheck: AclCheck = AclSimpleCheck((user, AclAddress.fromProject(proj.ref), permissions)).accepted + // TODO this dependency does nothing because we lookup using labels instead of UUIds in the tests... + // Does this endpoint need resolution by UUId? Do users need it? + val groupDirectives = DeltaSchemeDirectives(FetchContextDummy(Map(proj.ref -> proj.context))) + val identities = IdentitiesDummy(Caller(user, Set(user))) + Route.seal(BatchFilesRoutes(config, identities, aclCheck, batchFiles, groupDirectives, IndexingAction.noop)) + } + + def callBulkCopyEndpoint( + route: Route, + destProj: ProjectRef, + payload: Json, + user: User, + destStorageId: Option[IdSegment] = None, + destTag: Option[UserTag] = None + )(assert: => Assertion): Assertion = { + val asUser = addCredentials(OAuth2BearerToken(user.subject)) + val destStorageIdParam = destStorageId.map(id => s"storage=${id.asString}") + val destTagParam = destTag.map(tag => s"tag=${tag.value}") + val params = (destStorageIdParam.toList ++ destTagParam.toList).mkString("&") + Post(s"/v1/bulk/files/$destProj?$params", payload.toEntity()) ~> asUser ~> route ~> check(assert) + } + + def testBulkCopySucceedsForStubbedFiles( + sourceProj: Project, + sourceFileIds: NonEmptyList[FileId], + destStorageId: Option[IdSegment] = None, + destTag: Option[UserTag] = None + ): Assertion = { + val (destProj, user) = (genProject(), genUser(realm)) + val sourceFileResources = sourceFileIds.map(genFileResource(_, destProj.context)) + val events = ListBuffer.empty[BatchFilesCopyFilesCalled] + val stubbedBatchFiles = BatchFilesMock.withStubbedCopyFiles(sourceFileResources, events) + + val route = mkRoute(stubbedBatchFiles, sourceProj, user, Set(files.permissions.read)) + val payload = BatchFilesRoutesSpec.mkBulkCopyPayload(sourceProj.ref, sourceFileIds) + + callBulkCopyEndpoint(route, destProj.ref, payload, user, destStorageId, destTag) { + response.status shouldBe StatusCodes.Created + val expectedBatchFilesCall = + BatchFilesCopyFilesCalled.fromTestData( + destProj.ref, + sourceProj.ref, + sourceFileIds, + user, + destStorageId, + destTag + ) + events.toList shouldBe List(expectedBatchFilesCall) + response.asJson shouldBe expectedBulkCopyJson(sourceFileResources) + } + } + + def expectedBulkCopyJson(stubbedResources: NonEmptyList[FileResource]): Json = + Json.obj( + "@context" := List(Vocabulary.contexts.bulkOperation, Vocabulary.contexts.metadata, fileContexts.files), + "_results" := stubbedResources.map(expectedBulkOperationFileResourceJson) + ) + + def expectedBulkOperationFileResourceJson(res: FileResource): Json = + FilesRoutesSpec + .fileMetadata( + res.value.project, + res.id, + res.value.attributes, + res.value.storage, + res.value.storageType, + res.rev, + res.deprecated, + res.createdBy, + res.updatedBy + ) + .accepted + .mapObject(_.remove("@context")) +} + +object BatchFilesRoutesSpec { + def mkBulkCopyPayload(sourceProj: ProjectRef, sourceFileIds: NonEmptyList[FileId]): Json = + Json.obj("sourceProjectRef" := sourceProj.toString, "files" := mkSourceFilesPayload(sourceFileIds)) + + def mkSourceFilesPayload(sourceFileIds: NonEmptyList[FileId]): NonEmptyList[Json] = + sourceFileIds.map(id => mkSourceFilePayloadFromIdSegmentRef(id.id)) + + def mkSourceFilePayloadFromIdSegmentRef(id: IdSegmentRef): Json = id match { + case IdSegmentRef.Latest(value) => mkSourceFilePayload(value.asString, None, None) + case IdSegmentRef.Revision(value, rev) => mkSourceFilePayload(value.asString, Some(rev), None) + case IdSegmentRef.Tag(value, tag) => mkSourceFilePayload(value.asString, None, Some(tag.value)) + } + + def mkSourceFilePayload(id: String, rev: Option[Int], tag: Option[String]): Json = + Json.obj("sourceFileId" := id, "sourceRev" := rev, "sourceTag" := tag) +} diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala index cabb839c63..ffe77348d5 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/files/routes/FilesRoutesSpec.scala @@ -138,7 +138,6 @@ class FilesRoutesSpec )(uuidF, typedSystem) private val groupDirectives = DeltaSchemeDirectives(fetchContext, ioFromMap(uuid -> projectRef.organization), ioFromMap(uuid -> projectRef)) - private lazy val routes = routesWithIdentities(identities) private def routesWithIdentities(identities: Identities) = Route.seal(FilesRoutes(stCfg, identities, aclCheck, files, groupDirectives, IndexingAction.noop)) @@ -632,9 +631,11 @@ class FilesRoutesSpec } } - def givenAFile(test: String => Assertion): Assertion = { + def givenAFile(test: String => Assertion): Assertion = givenAFileInProject("org/proj")(test) + + def givenAFileInProject(projRef: String)(test: String => Assertion): Assertion = { val id = genString() - Put(s"/v1/files/org/proj/$id", entity(s"$id")) ~> asWriter ~> routes ~> check { + Put(s"/v1/files/$projRef/$id", entity(s"$id")) ~> asWriter ~> routes ~> check { status shouldEqual StatusCodes.Created } test(id) diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/RemoteStorageClientSpec.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/RemoteStorageClientSpec.scala index 365c8f1011..aba6c62fa8 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/RemoteStorageClientSpec.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/operations/remote/client/RemoteStorageClientSpec.scala @@ -86,7 +86,9 @@ class RemoteStorageClientSpec(docker: RemoteStorageDocker) } "move a file" in { - client.moveFile(bucket, Uri.Path("my/file-1.txt"), Uri.Path("other/file-1.txt"))(baseUri).accepted shouldEqual + client + .moveFile(bucket, Uri.Path("my/file-1.txt"), Uri.Path("other/file-1.txt"))(baseUri) + .accepted shouldEqual attributes.copy( location = s"file:///app/$BucketName/nexus/other/file-1.txt", digest = NotComputedDigest diff --git a/delta/rdf/src/main/scala/ch/epfl/bluebrain/nexus/delta/rdf/Vocabulary.scala b/delta/rdf/src/main/scala/ch/epfl/bluebrain/nexus/delta/rdf/Vocabulary.scala index 327fff5517..cfefe7b018 100644 --- a/delta/rdf/src/main/scala/ch/epfl/bluebrain/nexus/delta/rdf/Vocabulary.scala +++ b/delta/rdf/src/main/scala/ch/epfl/bluebrain/nexus/delta/rdf/Vocabulary.scala @@ -216,6 +216,7 @@ object Vocabulary { val acls = contexts + "acls.json" val aclsMetadata = contexts + "acls-metadata.json" + val bulkOperation = contexts + "bulk-operation.json" val error = contexts + "error.json" val identities = contexts + "identities.json" val metadata = contexts + "metadata.json" diff --git a/delta/sdk/src/main/resources/contexts/bulk-operation.json b/delta/sdk/src/main/resources/contexts/bulk-operation.json new file mode 100644 index 0000000000..6be92c1da4 --- /dev/null +++ b/delta/sdk/src/main/resources/contexts/bulk-operation.json @@ -0,0 +1,9 @@ +{ + "@context": { + "_results": { + "@id": "https://bluebrain.github.io/nexus/vocabulary/results", + "@container": "@list" + } + }, + "@id": "https://bluebrain.github.io/nexus/contexts/bulk-operation.json" +} \ No newline at end of file diff --git a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/jsonld/BulkOperationResults.scala b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/jsonld/BulkOperationResults.scala new file mode 100644 index 0000000000..da62735072 --- /dev/null +++ b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/jsonld/BulkOperationResults.scala @@ -0,0 +1,24 @@ +package ch.epfl.bluebrain.nexus.delta.sdk.jsonld + +import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.{contexts, nxv} +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.ContextValue +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.encoder.JsonLdEncoder +import io.circe.syntax._ +import io.circe.{Encoder, Json, JsonObject} + +final case class BulkOperationResults[A](results: Seq[A]) + +object BulkOperationResults { + + private val context = ContextValue(contexts.bulkOperation, contexts.metadata) + + implicit def encoder[A: Encoder.AsObject]: Encoder.AsObject[BulkOperationResults[A]] = + Encoder.AsObject.instance { r => + JsonObject(nxv.results.prefix -> Json.fromValues(r.results.map(_.asJson))) + } + + def searchResultsJsonLdEncoder[A: Encoder.AsObject]( + additionalContext: ContextValue + ): JsonLdEncoder[BulkOperationResults[A]] = + JsonLdEncoder.computeFromCirce(context.merge(additionalContext)) +} diff --git a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/model/ResourceF.scala b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/model/ResourceF.scala index 0f72f1c6ee..369cee2486 100644 --- a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/model/ResourceF.scala +++ b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/model/ResourceF.scala @@ -177,7 +177,7 @@ object ResourceF { implicit private def metadataJsonLdEncoder(implicit base: BaseUri): JsonLdEncoder[ResourceMetadata] = JsonLdEncoder.computeFromCirce(BNode.random, ContextValue(contexts.metadata)) - implicit def resourceFEncoder[A: Encoder.AsObject](implicit base: BaseUri): Encoder.AsObject[ResourceF[A]] = + implicit def resourceFEncoderObj[A: Encoder.AsObject](implicit base: BaseUri): Encoder.AsObject[ResourceF[A]] = Encoder.encodeJsonObject.contramapObject { r => ResourceIdAndTypes(r.resolvedId, r.types).asJsonObject deepMerge r.value.asJsonObject deepMerge diff --git a/docs/src/main/paradox/docs/delta/api/files-api.md b/docs/src/main/paradox/docs/delta/api/files-api.md index 1d326b3a7b..b5fe65cbe5 100644 --- a/docs/src/main/paradox/docs/delta/api/files-api.md +++ b/docs/src/main/paradox/docs/delta/api/files-api.md @@ -106,7 +106,7 @@ POST /v1/files/{org_label}/{project_label}?storage={storageId}&tag={tagName} When not specified, the default storage of the project is used. - `{path}`: String - the relative location (from the point of view of storage folder) on the remote storage where the file exists. - `{filename}`: String - the name that will be given to the file during linking. This field is optional. When not specified, the original filename is retained. -- `{mediaType}`: String - the MediaType fo the file. This field is optional. When not specified, Nexus Delta will attempt to detectput +- `{mediaType}`: String - the MediaType fo the file. This field is optional. When not specified, Nexus Delta will attempt to detect it. - `{tagName}` an optional label given to the linked file resource on its first revision. **Example** diff --git a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/Main.scala b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/Main.scala index ee3c07ed0b..74e66b9e2b 100644 --- a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/Main.scala +++ b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/Main.scala @@ -6,12 +6,13 @@ import akka.http.scaladsl.Http import akka.http.scaladsl.server.Route import akka.util.Timeout import cats.effect.{ExitCode, IO, IOApp} +import ch.epfl.bluebrain.nexus.delta.kernel.utils.TransactionalFileCopier import ch.epfl.bluebrain.nexus.storage.Storages.DiskStorage import ch.epfl.bluebrain.nexus.storage.attributes.{AttributesCache, ContentTypeDetector} import ch.epfl.bluebrain.nexus.storage.auth.AuthorizationMethod import ch.epfl.bluebrain.nexus.storage.config.AppConfig._ import ch.epfl.bluebrain.nexus.storage.config.{AppConfig, Settings} -import ch.epfl.bluebrain.nexus.storage.files.{CopyFiles, ValidateFile} +import ch.epfl.bluebrain.nexus.storage.files.ValidateFile import ch.epfl.bluebrain.nexus.storage.routes.Routes import com.typesafe.config.{Config, ConfigFactory} import kamon.Kamon @@ -60,9 +61,9 @@ object Main extends IOApp { implicit val clock = Clock.systemUTC implicit val contentTypeDetector = new ContentTypeDetector(appConfig.mediaTypeDetector) - val attributesCache: AttributesCache = AttributesCache[AkkaSource] - val validateFile: ValidateFile = ValidateFile.mk(appConfig.storage) - val copyFiles: CopyFiles = CopyFiles.mk() + val attributesCache: AttributesCache = AttributesCache[AkkaSource] + val validateFile: ValidateFile = ValidateFile.mk(appConfig.storage) + val copyFiles: TransactionalFileCopier = TransactionalFileCopier.mk() val storages: Storages[AkkaSource] = new DiskStorage( diff --git a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/StorageError.scala b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/StorageError.scala index e04ec5ddce..471f50bef9 100644 --- a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/StorageError.scala +++ b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/StorageError.scala @@ -2,7 +2,6 @@ package ch.epfl.bluebrain.nexus.storage import akka.http.scaladsl.model.StatusCodes import akka.http.scaladsl.model.Uri.Path -import ch.epfl.bluebrain.nexus.storage.files.CopyBetween import ch.epfl.bluebrain.nexus.storage.routes.StatusFrom import io.circe.generic.extras.Configuration import io.circe.generic.extras.semiauto.deriveConfiguredEncoder @@ -96,11 +95,6 @@ object StorageError { */ final case class OperationTimedOut(override val msg: String) extends StorageError(msg) - final case class CopyOperationFailed(failingCopy: CopyBetween) - extends StorageError( - s"Copy operation failed from source ${failingCopy.source} to destination ${failingCopy.destination}." - ) - @nowarn("cat=unused") implicit private val config: Configuration = Configuration.default.withDiscriminator("@type") diff --git a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/Storages.scala b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/Storages.scala index e0713211ed..67c880f792 100644 --- a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/Storages.scala +++ b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/Storages.scala @@ -6,6 +6,7 @@ import akka.stream.alpakka.file.scaladsl.Directory import akka.stream.scaladsl.{FileIO, Keep} import cats.data.{EitherT, NonEmptyList} import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.kernel.utils.{CopyBetween, TransactionalFileCopier} import ch.epfl.bluebrain.nexus.storage.File._ import ch.epfl.bluebrain.nexus.storage.Rejection.PathNotFound import ch.epfl.bluebrain.nexus.storage.StorageError.{InternalError, PermissionsFixingFailed} @@ -15,11 +16,12 @@ import ch.epfl.bluebrain.nexus.storage.Storages.{BucketExistence, PathExistence} import ch.epfl.bluebrain.nexus.storage.attributes.AttributesComputation._ import ch.epfl.bluebrain.nexus.storage.attributes.{AttributesCache, ContentTypeDetector} import ch.epfl.bluebrain.nexus.storage.config.AppConfig.{DigestConfig, StorageConfig} -import ch.epfl.bluebrain.nexus.storage.files.{CopyFileOutput, CopyFiles, ValidateFile} +import ch.epfl.bluebrain.nexus.storage.files.{CopyFileOutput, ValidateFile} import ch.epfl.bluebrain.nexus.storage.routes.CopyFile import java.nio.file.StandardCopyOption._ import java.nio.file.{Files, Path} +import fs2.io.file.{Path => Fs2Path} import java.security.MessageDigest import scala.concurrent.{ExecutionContext, Future} import scala.sys.process._ @@ -128,15 +130,21 @@ trait Storages[Source] { object Storages { - sealed trait BucketExistence - sealed trait PathExistence { + sealed trait BucketExistence { + def exists: Boolean + } + sealed trait PathExistence { def exists: Boolean } object BucketExistence { - final case object BucketExists extends BucketExistence - final case object BucketDoesNotExist extends BucketExistence - type BucketExists = BucketExists.type + final case object BucketExists extends BucketExistence { + val exists = true + } + final case object BucketDoesNotExist extends BucketExistence { + val exists = false + } + type BucketExists = BucketExists.type type BucketDoesNotExist = BucketDoesNotExist.type } @@ -160,7 +168,7 @@ object Storages { digestConfig: DigestConfig, cache: AttributesCache, validateFile: ValidateFile, - copyFiles: CopyFiles + copyFiles: TransactionalFileCopier )(implicit ec: ExecutionContext, mt: Materializer @@ -263,12 +271,17 @@ object Storages { IO.raiseError(InternalError(s"Path '$absPath' is not a file nor a directory")) def copyFiles( - name: String, + destBucket: String, files: NonEmptyList[CopyFile] )(implicit bucketEv: BucketExists, pathEv: PathDoesNotExist): IO[RejOr[NonEmptyList[CopyFileOutput]]] = (for { - validated <- files.traverse(f => EitherT(validateFile.forCopyWithinProtectedDir(name, f.source, f.destination))) - _ <- EitherT.right[Rejection](copyFiles.copyValidated(validated)) + validated <- + files.traverse(f => + EitherT(validateFile.forCopyWithinProtectedDir(f.sourceBucket, destBucket, f.source, f.destination)) + ) + copyBetween = + validated.map(v => CopyBetween(Fs2Path.fromNioPath(v.absSourcePath), Fs2Path.fromNioPath(v.absDestPath))) + _ <- EitherT.right[Rejection](copyFiles.copyAll(copyBetween)) } yield files.zip(validated).map { case (raw, valid) => CopyFileOutput(raw.source, raw.destination, valid.absSourcePath, valid.absDestPath) }).value diff --git a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/files/CopyBetween.scala b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/files/CopyBetween.scala deleted file mode 100644 index 38aa68ca04..0000000000 --- a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/files/CopyBetween.scala +++ /dev/null @@ -1,12 +0,0 @@ -package ch.epfl.bluebrain.nexus.storage.files - -import fs2.io.file.Path -import io.circe.Encoder -import ch.epfl.bluebrain.nexus.storage._ -import io.circe.generic.semiauto.deriveEncoder - -final case class CopyBetween(source: Path, destination: Path) - -object CopyBetween { - implicit val enc: Encoder[CopyBetween] = deriveEncoder -} diff --git a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/files/ValidateFile.scala b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/files/ValidateFile.scala index ac005b0302..312a292bf1 100644 --- a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/files/ValidateFile.scala +++ b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/files/ValidateFile.scala @@ -25,7 +25,8 @@ trait ValidateFile { ): IO[RejOr[ValidatedMoveFile]] def forCopyWithinProtectedDir( - name: String, + sourceBucket: String, + destBucket: String, sourcePath: Uri.Path, destPath: Uri.Path ): IO[RejOr[ValidatedCopyFile]] @@ -33,7 +34,12 @@ trait ValidateFile { sealed abstract case class ValidatedCreateFile(absDestPath: Path) sealed abstract case class ValidatedMoveFile(name: String, absSourcePath: Path, absDestPath: Path, isDir: Boolean) -sealed abstract case class ValidatedCopyFile(name: String, absSourcePath: Path, absDestPath: Path) +sealed abstract case class ValidatedCopyFile( + sourceBucket: String, + destBucket: String, + absSourcePath: Path, + absDestPath: Path +) object ValidateFile { @@ -71,25 +77,27 @@ object ValidateFile { } override def forCopyWithinProtectedDir( - name: String, + sourceBucket: String, + destBucket: String, sourcePath: Uri.Path, destPath: Uri.Path ): IO[RejOr[ValidatedCopyFile]] = { - val bucketProtectedPath = basePath(config, name) - val absSourcePath = filePath(config, name, sourcePath) - val absDestPath = filePath(config, name, destPath) + val sourceBucketProtectedPath = basePath(config, sourceBucket) + val destBucketProtectedPath = basePath(config, destBucket) + val absSourcePath = filePath(config, sourceBucket, sourcePath) + val absDestPath = filePath(config, destBucket, destPath) - def notFound = PathNotFound(name, sourcePath) + def notFound = PathNotFound(destBucket, sourcePath) (for { _ <- rejectIf(fileExists(absSourcePath).map(!_), notFound) - _ <- rejectIf((!descendantOf(absSourcePath, bucketProtectedPath)).pure[IO], notFound) - _ <- throwIf(!descendantOf(absDestPath, bucketProtectedPath), PathInvalid(name, destPath)) - _ <- rejectIf(fileExists(absDestPath), PathAlreadyExists(name, destPath)) + _ <- rejectIf((!descendantOf(absSourcePath, sourceBucketProtectedPath)).pure[IO], notFound) + _ <- throwIf(!descendantOf(absDestPath, destBucketProtectedPath), PathInvalid(destBucket, destPath)) + _ <- rejectIf(fileExists(absDestPath), PathAlreadyExists(destBucket, destPath)) isFile <- EitherT.right[Rejection](isRegularFile(absSourcePath)) - _ <- throwIf(!isFile, PathInvalid(name, sourcePath)) - } yield new ValidatedCopyFile(name, absSourcePath, absDestPath) {}).value + _ <- throwIf(!isFile, PathInvalid(sourceBucket, sourcePath)) + } yield new ValidatedCopyFile(sourceBucket, destBucket, absSourcePath, absDestPath) {}).value } def fileExists(absSourcePath: Path): IO[Boolean] = IO.blocking(Files.exists(absSourcePath)) diff --git a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/routes/CopyFile.scala b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/routes/CopyFile.scala index b794d33486..97dc73994b 100644 --- a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/routes/CopyFile.scala +++ b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/routes/CopyFile.scala @@ -5,7 +5,7 @@ import ch.epfl.bluebrain.nexus.storage._ import io.circe.Decoder import io.circe.generic.semiauto.deriveDecoder -final case class CopyFile(source: Uri.Path, destination: Uri.Path) +final case class CopyFile(sourceBucket: String, source: Uri.Path, destination: Uri.Path) object CopyFile { implicit val dec: Decoder[CopyFile] = deriveDecoder } diff --git a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/routes/StorageDirectives.scala b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/routes/StorageDirectives.scala index 25198e131d..2014b28759 100644 --- a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/routes/StorageDirectives.scala +++ b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/routes/StorageDirectives.scala @@ -39,11 +39,11 @@ object StorageDirectives { def validatePath(name: String, path: Path): Directive0 = if (pathInvalid(path)) failWith(PathInvalid(name, path)) else pass - def validatePaths(name: String, paths: NonEmptyList[Path]): Directive0 = - paths + def validatePaths(pathsByBucket: NonEmptyList[(String, Path)]): Directive0 = + pathsByBucket .collectFirst[Directive0] { - case p if pathInvalid(p) => - failWith(PathInvalid(name, p)) + case (bucket, p) if pathInvalid(p) => + failWith(PathInvalid(bucket, p)) } .getOrElse(pass) @@ -71,6 +71,13 @@ object StorageDirectives { case _ => reject(BucketNotFound(name)) } + def bucketsExist(buckets: NonEmptyList[String])(implicit storages: Storages[_]): Directive1[BucketExists] = + buckets + .map(storages.exists) + .zip(buckets) + .collectFirst[Directive1[BucketExists]] { case (e, bucket) if !e.exists => reject(BucketNotFound(bucket)) } + .getOrElse(provide(BucketExists)) + /** * Returns the evidence that a path exists * diff --git a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/routes/StorageRoutes.scala b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/routes/StorageRoutes.scala index 8451dbbaf5..7ca6accc78 100644 --- a/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/routes/StorageRoutes.scala +++ b/storage/src/main/scala/ch/epfl/bluebrain/nexus/storage/routes/StorageRoutes.scala @@ -8,6 +8,7 @@ import akka.http.scaladsl.server.Route import cats.data.NonEmptyList import cats.effect.unsafe.implicits._ import cats.syntax.all._ +import ch.epfl.bluebrain.nexus.delta.kernel.Logger import ch.epfl.bluebrain.nexus.storage.File.{Digest, FileAttributes} import ch.epfl.bluebrain.nexus.storage.config.AppConfig import ch.epfl.bluebrain.nexus.storage.config.AppConfig.HttpConfig @@ -22,6 +23,8 @@ import kamon.instrumentation.akka.http.TracingDirectives.operationName class StorageRoutes()(implicit storages: Storages[AkkaSource], hc: HttpConfig) { + private val logger = Logger[StorageRoutes] + def routes: Route = // Consume buckets/{name}/ (encodeResponse & pathPrefix("buckets" / Segment)) { name => @@ -72,12 +75,17 @@ class StorageRoutes()(implicit storages: Storages[AkkaSource], hc: HttpConfig) { }, operationName(s"/${hc.prefix}/buckets/{}/files") { post { - // Copy files within protected directory + // Copy files within protected directory between potentially different buckets entity(as[CopyFilePayload]) { payload => val files = payload.files - pathsDoNotExist(name, files.map(_.destination)).apply { implicit pathNotExistEvidence => - validatePaths(name, files.map(_.source)) { - complete(storages.copyFiles(name, files).runWithStatus(Created)) + bucketsExist(files.map(_.sourceBucket)).apply { implicit bucketExistsEvidence => + pathsDoNotExist(name, files.map(_.destination)).apply { implicit pathNotExistEvidence => + validatePaths(files.map(c => c.sourceBucket -> c.source)) { + complete( + (logger.info(s"Received request to copy files: $files") >> + storages.copyFiles(name, files)).runWithStatus(Created) + ) + } } } } diff --git a/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/DiskStorageSpec.scala b/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/DiskStorageSpec.scala index 21046d6bd9..f9c1d33afb 100644 --- a/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/DiskStorageSpec.scala +++ b/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/DiskStorageSpec.scala @@ -10,6 +10,7 @@ import akka.util.ByteString import cats.data.NonEmptyList import cats.effect.IO import ch.epfl.bluebrain.nexus.delta.kernel.http.MediaTypeDetectorConfig +import ch.epfl.bluebrain.nexus.delta.kernel.utils.TransactionalFileCopier import ch.epfl.bluebrain.nexus.storage.File.{Digest, FileAttributes} import ch.epfl.bluebrain.nexus.storage.Rejection.{PathAlreadyExists, PathNotFound} import ch.epfl.bluebrain.nexus.storage.StorageError.{PathInvalid, PermissionsFixingFailed} @@ -18,7 +19,7 @@ import ch.epfl.bluebrain.nexus.storage.Storages.DiskStorage import ch.epfl.bluebrain.nexus.storage.Storages.PathExistence.{PathDoesNotExist, PathExists} import ch.epfl.bluebrain.nexus.storage.attributes.{AttributesCache, ContentTypeDetector} import ch.epfl.bluebrain.nexus.storage.config.AppConfig.{DigestConfig, StorageConfig} -import ch.epfl.bluebrain.nexus.storage.files.{CopyFileOutput, CopyFiles, ValidateFile} +import ch.epfl.bluebrain.nexus.storage.files.{CopyFileOutput, ValidateFile} import ch.epfl.bluebrain.nexus.storage.routes.CopyFile import ch.epfl.bluebrain.nexus.storage.utils.Randomness import ch.epfl.bluebrain.nexus.testkit.scalatest.ce.CatsEffectSpec @@ -51,7 +52,7 @@ class DiskStorageSpec val contentTypeDetector = new ContentTypeDetector(MediaTypeDetectorConfig.Empty) val cache = mock[AttributesCache] val validateFile = ValidateFile.mk(sConfig) - val copyFiles = CopyFiles.mk() + val copyFiles = TransactionalFileCopier.mk() val storage = new DiskStorage(sConfig, contentTypeDetector, dConfig, cache, validateFile, copyFiles) override def afterAll(): Unit = { @@ -336,7 +337,7 @@ class DiskStorageSpec "fail when source does not exists" in new AbsoluteDirectoryCreated { val source = randomString() - val files = NonEmptyList.of(CopyFile(Uri.Path(source), Uri.Path(randomString()))) + val files = NonEmptyList.of(CopyFile(name, Uri.Path(source), Uri.Path(randomString()))) storage.copyFiles(name, files).accepted.leftValue shouldEqual PathNotFound(name, Uri.Path(source)) } @@ -345,7 +346,7 @@ class DiskStorageSpec val absoluteFile = baseRootPath.resolve(Paths.get(file)) Files.createDirectories(absoluteFile.getParent) Files.write(absoluteFile, "something".getBytes(StandardCharsets.UTF_8)) - val files = NonEmptyList.of(CopyFile(Uri.Path(file), Uri.Path(randomString()))) + val files = NonEmptyList.of(CopyFile(name, Uri.Path(file), Uri.Path(randomString()))) storage.copyFiles(name, files).accepted.leftValue shouldEqual PathNotFound(name, Uri.Path(file)) } @@ -359,7 +360,7 @@ class DiskStorageSpec val destFile = "destFile.txt" val resolvedDestFile = basePath.resolve(Paths.get(destFile)) Files.write(resolvedDestFile, "somethingelse".getBytes(StandardCharsets.UTF_8)) - val files = NonEmptyList.of(CopyFile(Uri.Path(file), Uri.Path(destFile))) + val files = NonEmptyList.of(CopyFile(name, Uri.Path(file), Uri.Path(destFile))) storage.copyFiles(name, files).accepted.leftValue shouldEqual PathAlreadyExists(name, Uri.Path(destFile)) } @@ -372,14 +373,14 @@ class DiskStorageSpec val content = "some content" Files.write(absoluteFile, content.getBytes(StandardCharsets.UTF_8)) - val files = NonEmptyList.of(CopyFile(Uri.Path(file), dest)) + val files = NonEmptyList.of(CopyFile(name, Uri.Path(file), dest)) storage.copyFiles(name, files).rejectedWith[StorageError] shouldEqual PathInvalid(name, dest) Files.exists(absoluteFile) shouldEqual true } - "pass on file specified for absolute/relative path" in { + "pass on file in same bucket specified for absolute/relative path" in { forAll(List(true, false)) { useRelativePath => new AbsoluteDirectoryCreated { val sourceFile = sConfig.protectedDirectory.toString + "/my !file.txt" @@ -393,7 +394,7 @@ class DiskStorageSpec val absoluteDestFile = basePath.resolve(Paths.get(destPath)) val sourcePathToUse = if (useRelativePath) sourceFile else absoluteSourceFile.toString - val files = NonEmptyList.of(CopyFile(Uri.Path(sourcePathToUse), Uri.Path(destPath))) + val files = NonEmptyList.of(CopyFile(name, Uri.Path(sourcePathToUse), Uri.Path(destPath))) val expectedOutput = CopyFileOutput(Uri.Path(sourcePathToUse), Uri.Path(destPath), absoluteSourceFile, absoluteDestFile) @@ -406,6 +407,35 @@ class DiskStorageSpec } } + "pass on file in different bucket specified for absolute/relative path" in { + forAll(List(true, false)) { useRelativePath => + val dest: AbsoluteDirectoryCreated = new AbsoluteDirectoryCreated {} + val source: AbsoluteDirectoryCreated = new AbsoluteDirectoryCreated {} + val sourceFile = sConfig.protectedDirectory.toString + "/my !file.txt" + val absoluteSourceFile = source.basePath.resolve(Paths.get(sourceFile)) + Files.createDirectories(absoluteSourceFile.getParent) + + val content = "some content" + Files.write(absoluteSourceFile, content.getBytes(StandardCharsets.UTF_8)) + + val destPath = "some/other path.txt" + val absoluteDestFile = dest.basePath.resolve(Paths.get(destPath)) + + val sourcePathToUse = if (useRelativePath) sourceFile else absoluteSourceFile.toString + val files = NonEmptyList.of(CopyFile(source.name, Uri.Path(sourcePathToUse), Uri.Path(destPath))) + val expectedOutput = + CopyFileOutput(Uri.Path(sourcePathToUse), Uri.Path(destPath), absoluteSourceFile, absoluteDestFile) + + Files.exists(absoluteSourceFile) shouldEqual true + + storage.copyFiles(dest.name, files).accepted shouldEqual Right(NonEmptyList.of(expectedOutput)) + + Files.exists(absoluteSourceFile) shouldEqual true + Files.exists(absoluteDestFile) shouldEqual true + Files.readString(absoluteDestFile, StandardCharsets.UTF_8) shouldEqual content + } + } + } "fetching" should { diff --git a/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/routes/StorageRoutesSpec.scala b/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/routes/StorageRoutesSpec.scala index b3e8a1f1ed..89cb7a6f66 100644 --- a/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/routes/StorageRoutesSpec.scala +++ b/storage/src/test/scala/ch/epfl/bluebrain/nexus/storage/routes/StorageRoutesSpec.scala @@ -290,10 +290,11 @@ class StorageRoutesSpec "copying a file" should { - "fail when bucket does not exist" in new Ctx { + "fail when destination bucket does not exist" in new Ctx { storages.exists(name) shouldReturn BucketDoesNotExist - val json = Json.arr(Json.obj("source" := "source/dir", "destination" := "/path/to/myfile.txt")) + val json = + Json.arr(Json.obj("sourceBucket" := name, "source" := "source/dir", "destination" := "/path/to/myfile.txt")) Post(s"/v1/buckets/$name/files", json) ~> route ~> check { status shouldEqual NotFound @@ -308,6 +309,29 @@ class StorageRoutesSpec } } + "fail when a source bucket does not exist" in new Ctx { + val sourceBucket = randomString() + storages.exists(name) shouldReturn BucketExists + storages.exists(sourceBucket) shouldReturn BucketDoesNotExist + + val json = Json.arr( + Json.obj("sourceBucket" := sourceBucket, "source" := "source/dir", "destination" := "/path/to/myfile.txt") + ) + + Post(s"/v1/buckets/$name/files", json) ~> route ~> check { + status shouldEqual NotFound + responseAs[Json] shouldEqual jsonContentOf( + "/error.json", + Map( + quote("{type}") -> "BucketNotFound", + quote("{reason}") -> s"The provided bucket '$sourceBucket' does not exist." + ) + ) + storages.exists(name) wasCalled once + storages.exists(sourceBucket) wasCalled once + } + } + "fail if an empty array is passed" in new Ctx { storages.exists(name) shouldReturn BucketExists @@ -322,11 +346,11 @@ class StorageRoutesSpec val source = "source/dir" val dest = "dest/dir" storages.pathExists(name, Uri.Path(dest)) shouldReturn PathDoesNotExist - val input = NonEmptyList.of(CopyFile(Uri.Path(source), Uri.Path(dest))) + val input = NonEmptyList.of(CopyFile(name, Uri.Path(source), Uri.Path(dest))) storages.copyFiles(name, input)(BucketExists, PathDoesNotExist) shouldReturn IO.raiseError(InternalError("something went wrong")) - val json = Json.arr(Json.obj("source" := source, "destination" := dest)) + val json = Json.arr(Json.obj("sourceBucket" := name, "source" := source, "destination" := dest)) Post(s"/v1/buckets/$name/files", json) ~> route ~> check { status shouldEqual InternalServerError @@ -347,7 +371,7 @@ class StorageRoutesSpec val dest = "dest/dir" storages.pathExists(name, Uri.Path(dest)) shouldReturn PathDoesNotExist - val json = Json.arr(Json.obj("source" := source, "destination" := dest)) + val json = Json.arr(Json.obj("sourceBucket" := name, "source" := source, "destination" := dest)) Post(s"/v1/buckets/$name/files", json) ~> route ~> check { status shouldEqual BadRequest @@ -364,19 +388,21 @@ class StorageRoutesSpec } "pass" in new Ctx { + val sourceBucket = randomString() storages.exists(name) shouldReturn BucketExists - val source = "source/dir" - val dest = "dest/dir" - val output = + storages.exists(sourceBucket) shouldReturn BucketExists + val source = "source/dir" + val dest = "dest/dir" + val output = CopyFileOutput(Uri.Path(source), Uri.Path(dest), Paths.get(s"/rootdir/$source"), Paths.get(s"/rootdir/$dest")) storages.pathExists(name, Uri.Path(dest)) shouldReturn PathDoesNotExist - storages.copyFiles(name, NonEmptyList.of(CopyFile(Uri.Path(source), Uri.Path(dest))))( + storages.copyFiles(name, NonEmptyList.of(CopyFile(sourceBucket, Uri.Path(source), Uri.Path(dest))))( BucketExists, PathDoesNotExist ) shouldReturn IO.pure(Right(NonEmptyList.of(output))) - val json = Json.arr(Json.obj("source" := source, "destination" := dest)) + val json = Json.arr(Json.obj("sourceBucket" := sourceBucket, "source" := source, "destination" := dest)) val response = Json.arr( Json.obj( "sourcePath" := source, @@ -390,7 +416,7 @@ class StorageRoutesSpec status shouldEqual Created responseAs[Json] shouldEqual response - storages.copyFiles(name, NonEmptyList.of(CopyFile(Uri.Path(source), Uri.Path(dest))))( + storages.copyFiles(name, NonEmptyList.of(CopyFile(sourceBucket, Uri.Path(source), Uri.Path(dest))))( BucketExists, PathDoesNotExist ) wasCalled once @@ -406,8 +432,8 @@ class StorageRoutesSpec storages.pathExists(name, Uri.Path(existingDest)) shouldReturn PathExists val json = Json.arr( - Json.obj("source" := source, "destination" := dest), - Json.obj("source" := source, "destination" := existingDest) + Json.obj("sourceBucket" := name, "source" := source, "destination" := dest), + Json.obj("sourceBucket" := name, "source" := source, "destination" := existingDest) ) Post(s"/v1/buckets/$name/files", json) ~> route ~> check { @@ -423,8 +449,8 @@ class StorageRoutesSpec storages.pathExists(name, Uri.Path(dest)) shouldReturn PathDoesNotExist val json = Json.arr( - Json.obj("source" := source, "destination" := dest), - Json.obj("source" := invalidSource, "destination" := dest) + Json.obj("sourceBucket" := name, "source" := source, "destination" := dest), + Json.obj("sourceBucket" := name, "source" := invalidSource, "destination" := dest) ) Post(s"/v1/buckets/$name/files", json) ~> route ~> check { diff --git a/tests/src/test/resources/kg/storages/disk-perms.json b/tests/src/test/resources/kg/storages/disk-perms.json index 7f7a6ebc15..9f17d640c6 100644 --- a/tests/src/test/resources/kg/storages/disk-perms.json +++ b/tests/src/test/resources/kg/storages/disk-perms.json @@ -1,8 +1,8 @@ { - "@id": "https://bluebrain.github.io/nexus/vocabulary/mystorage2", + "@id": "https://bluebrain.github.io/nexus/vocabulary/{{id}}", "@type": "DiskStorage", "volume": "/default-volume", "default": false, - "readPermission": "disk/read", - "writePermission": "disk/write" + "readPermission": "{{read}}", + "writePermission": "{{write}}" } \ No newline at end of file diff --git a/tests/src/test/resources/kg/storages/disk.json b/tests/src/test/resources/kg/storages/disk.json index e9b286f0e4..a2451b08a1 100644 --- a/tests/src/test/resources/kg/storages/disk.json +++ b/tests/src/test/resources/kg/storages/disk.json @@ -1,5 +1,5 @@ { - "@id": "https://bluebrain.github.io/nexus/vocabulary/mystorage", + "@id": "https://bluebrain.github.io/nexus/vocabulary/{{id}}", "@type": "DiskStorage", "volume": "/default-volume", "default": false diff --git a/tests/src/test/resources/kg/storages/storage-source.json b/tests/src/test/resources/kg/storages/storage-source.json index e589fe675b..0b32544327 100644 --- a/tests/src/test/resources/kg/storages/storage-source.json +++ b/tests/src/test/resources/kg/storages/storage-source.json @@ -1,5 +1,5 @@ { - "@id" : "https://bluebrain.github.io/nexus/vocabulary/myexternalstorage", + "@id" : "https://bluebrain.github.io/nexus/vocabulary/{{id}}", "@type" : "RemoteDiskStorage", "default" : false, "endpoint" : "{{storageBase}}", diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/HttpClient.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/HttpClient.scala index 0ef05df433..fa60adb097 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/HttpClient.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/HttpClient.scala @@ -66,6 +66,11 @@ class HttpClient private (baseUrl: Uri, httpExt: HttpExt)(implicit )(implicit um: FromEntityUnmarshaller[A]): IO[Assertion] = requestAssert(PUT, url, Some(body), identity, extraHeaders)(assertResponse) + def postAndReturn[A](url: String, body: Json, identity: Identity, extraHeaders: Seq[HttpHeader] = jsonHeaders)( + assertResponse: (A, HttpResponse) => (A, Assertion) + )(implicit um: FromEntityUnmarshaller[A]): IO[A] = + requestAssertAndReturn(POST, url, Some(body), identity, extraHeaders)(assertResponse).map(_._1) + /** Put with no body */ def putEmptyBody[A](url: String, identity: Identity, extraHeaders: Seq[HttpHeader] = jsonHeaders)( assertResponse: (A, HttpResponse) => Assertion @@ -157,25 +162,25 @@ class HttpClient private (baseUrl: Uri, httpExt: HttpExt)(implicit )(implicit um: FromEntityUnmarshaller[A]): IO[Assertion] = requestAssert(DELETE, url, None, identity, extraHeaders)(assertResponse) - def requestAssert[A]( + def requestAssertAndReturn[A]( method: HttpMethod, url: String, body: Option[Json], identity: Identity, extraHeaders: Seq[HttpHeader] = jsonHeaders - )(assertResponse: (A, HttpResponse) => Assertion)(implicit um: FromEntityUnmarshaller[A]): IO[Assertion] = { + )(assertResponse: (A, HttpResponse) => (A, Assertion))(implicit um: FromEntityUnmarshaller[A]): IO[(A, Assertion)] = { def buildClue(a: A, response: HttpResponse) = s""" - |Endpoint: ${method.value} $url - |Identity: $identity - |Token: ${Option(tokensMap.get(identity)).map(_.credentials.token()).getOrElse("None")} - |Status code: ${response.status} - |Body: ${body.getOrElse("None")} - |Response: - |$a - |""".stripMargin - - requestJson( + |Endpoint: ${method.value} $url + |Identity: $identity + |Token: ${Option(tokensMap.get(identity)).map(_.credentials.token()).getOrElse("None")} + |Status code: ${response.status} + |Body: ${body.getOrElse("None")} + |Response: + |$a + |""".stripMargin + + requestJson[A, (A, Assertion)]( method, url, body, @@ -185,6 +190,17 @@ class HttpClient private (baseUrl: Uri, httpExt: HttpExt)(implicit ) } + def requestAssert[A]( + method: HttpMethod, + url: String, + body: Option[Json], + identity: Identity, + extraHeaders: Seq[HttpHeader] = jsonHeaders + )(assertResponse: (A, HttpResponse) => Assertion)(implicit um: FromEntityUnmarshaller[A]): IO[Assertion] = + requestAssertAndReturn[A](method, url, body, identity, extraHeaders) { (a, resp) => + (a, assertResponse(a, resp)) + }.map(_._2) + def sparqlQuery[A](url: String, query: String, identity: Identity, extraHeaders: Seq[HttpHeader] = Nil)( assertResponse: (A, HttpResponse) => Assertion )(implicit um: FromEntityUnmarshaller[A]): IO[Assertion] = { diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/CopyFilesSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/CopyFilesSpec.scala new file mode 100644 index 0000000000..6b11f04b24 --- /dev/null +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/CopyFilesSpec.scala @@ -0,0 +1,128 @@ +package ch.epfl.bluebrain.nexus.tests.kg + +import akka.http.scaladsl.model.{ContentTypes, StatusCodes} +import akka.util.ByteString +import cats.effect.IO +import cats.implicits.toTraverseOps +import ch.epfl.bluebrain.nexus.delta.kernel.utils.UrlUtils +import ch.epfl.bluebrain.nexus.tests.HttpClient._ +import ch.epfl.bluebrain.nexus.tests.Identity.storages.Coyote +import ch.epfl.bluebrain.nexus.tests.Optics +import ch.epfl.bluebrain.nexus.tests.kg.CopyFilesSpec.{Response, StorageDetails} +import io.circe.syntax.KeyOps +import io.circe.{Decoder, DecodingFailure, Json, JsonObject} +import org.scalatest.Assertion + +trait CopyFilesSpec { self: StorageSpec => + + "Copying multiple files" should { + + "succeed for a project in the same organization" in { + givenANewProjectAndStorageInExistingOrg(orgId) { destStorage => + val existingFiles = List(emptyTextFile, updatedJsonFileWithContentType, textFileWithContentType) + copyFilesAndCheckSavedResourcesAndContents(projectRef, existingFiles, destStorage) + } + } + + "succeed for a project in a different organization" in { + givenANewOrgProjectAndStorage { destStorage => + val sourceFiles = List(emptyTextFile, updatedJsonFileWithContentType, textFileWithContentType) + copyFilesAndCheckSavedResourcesAndContents(projectRef, sourceFiles, destStorage) + } + } + + "succeed for source files in different storages within a project" in { + givenANewStorageInExistingProject(projectRef) { sourceStorage1 => + givenANewStorageInExistingProject(projectRef) { sourceStorage2 => + givenANewOrgProjectAndStorage { destStorage => + val (sourceFile1, sourceFile2) = (genTextFileInput(), genTextFileInput()) + val sourceFiles = List(sourceFile1, sourceFile2) + + for { + _ <- uploadFileToProjectStorage(sourceFile1, sourceStorage1.projRef, sourceStorage1.storageId, None)( + expectCreated + ) + _ <- uploadFileToProjectStorage(sourceFile2, sourceStorage2.projRef, sourceStorage2.storageId, None)( + expectCreated + ) + result <- copyFilesAndCheckSavedResourcesAndContents(projectRef, sourceFiles, destStorage) + } yield result + } + } + } + } + + } + + def genTextFileInput(): Input = Input(genId(), genString(), ContentTypes.`text/plain(UTF-8)`, genString()) + + def mkPayload(sourceProjRef: String, sourceFiles: List[Input]): Json = { + val sourcePayloads = sourceFiles.map(f => Json.obj("sourceFileId" := f.fileId)) + Json.obj("sourceProjectRef" := sourceProjRef, "files" := sourcePayloads) + } + + private def copyFilesAndCheckSavedResourcesAndContents( + sourceProjRef: String, + sourceFiles: List[Input], + destStorage: StorageDetails + ): IO[Assertion] = { + val destProjRef = destStorage.projRef + val payload = mkPayload(sourceProjRef, sourceFiles) + val uri = s"/bulk/files/$destProjRef?storage=nxv:${destStorage.storageId}" + + for { + response <- deltaClient.postAndReturn[Response](uri, payload, Coyote) { (json, response) => + (json, expectCreated(json, response)) + } + _ <- checkFileResourcesExist(destProjRef, response) + assertions <- checkFileContentsAreCopiedCorrectly(destProjRef, sourceFiles, response) + } yield assertions.head + } + + def checkFileContentsAreCopiedCorrectly(destProjRef: String, sourceFiles: List[Input], response: Response) = + response.ids.zip(sourceFiles).traverse { case (destId, Input(_, filename, contentType, contents)) => + deltaClient + .get[ByteString](s"/files/$destProjRef/${UrlUtils.encode(destId)}", Coyote, acceptAll) { + expectDownload(filename, contentType, contents) + } + } + + def checkFileResourcesExist(destProjRef: String, response: Response) = + response.ids.traverse { id => + deltaClient.get[Json](s"/files/$destProjRef/${UrlUtils.encode(id)}", Coyote) { (json, response) => + response.status shouldEqual StatusCodes.OK + Optics.`@id`.getOption(json) shouldEqual Some(id) + } + } + + def givenANewProjectAndStorageInExistingOrg(org: String)(test: StorageDetails => IO[Assertion]): IO[Assertion] = { + val proj = genId() + val projRef = s"$org/$proj" + createProjects(Coyote, org, proj) >> + givenANewStorageInExistingProject(projRef)(test) + } + + def givenANewStorageInExistingProject(projRef: String)(test: StorageDetails => IO[Assertion]): IO[Assertion] = { + val (storageId, storageName) = (genId(), genString()) + createStorages(projRef, storageId, storageName) >> + test(StorageDetails(projRef, storageId)) + } + + def givenANewOrgProjectAndStorage(test: StorageDetails => IO[Assertion]): IO[Assertion] = + givenANewProjectAndStorageInExistingOrg(genId())(test) +} + +object CopyFilesSpec { + final case class StorageDetails(projRef: String, storageId: String) + + final case class Response(ids: List[String]) + + object Response { + implicit val dec: Decoder[Response] = Decoder.instance { cur => + cur + .get[List[JsonObject]]("_results") + .flatMap(_.traverse(_.apply("@id").flatMap(_.asString).toRight(DecodingFailure("Missing id", Nil)))) + .map(Response(_)) + } + } +} diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/DiskStorageSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/DiskStorageSpec.scala index f4f3b64d69..4e9b626add 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/DiskStorageSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/DiskStorageSpec.scala @@ -8,7 +8,7 @@ import ch.epfl.bluebrain.nexus.tests.iam.types.Permission import io.circe.Json import org.scalatest.Assertion -class DiskStorageSpec extends StorageSpec { +class DiskStorageSpec extends StorageSpec with CopyFilesSpec { override def storageName: String = "disk" @@ -32,32 +32,35 @@ class DiskStorageSpec extends StorageSpec { ): _* ) - override def createStorages: IO[Assertion] = { - val payload = jsonContentOf("kg/storages/disk.json") - val payload2 = jsonContentOf("kg/storages/disk-perms.json") + override def createStorages(projectRef: String, storId: String, storName: String): IO[Assertion] = { + val payload = jsonContentOf("kg/storages/disk.json", "id" -> storId) + val storageId2 = s"${storId}2" + val storage2Read = s"$storName/read" + val storage2Write = s"$storName/write" + val payload2 = + jsonContentOf("kg/storages/disk-perms.json", "id" -> storageId2, "read" -> storage2Read, "write" -> storage2Write) for { - _ <- deltaClient.post[Json](s"/storages/$projectRef", payload, Coyote) { (_, response) => - response.status shouldEqual StatusCodes.Created - } - _ <- deltaClient.get[Json](s"/storages/$projectRef/nxv:$storageId", Coyote) { (json, response) => - val expected = storageResponse(projectRef, storageId, "resources/read", "files/write") - filterMetadataKeys(json) should equalIgnoreArrayOrder(expected) - response.status shouldEqual StatusCodes.OK - } - _ <- permissionDsl.addPermissions( - Permission(storageName, "read"), - Permission(storageName, "write") - ) - _ <- deltaClient.post[Json](s"/storages/$projectRef", payload2, Coyote) { (_, response) => - response.status shouldEqual StatusCodes.Created - } - storageId2 = s"${storageId}2" - _ <- deltaClient.get[Json](s"/storages/$projectRef/nxv:$storageId2", Coyote) { (json, response) => - val expected = storageResponse(projectRef, storageId2, s"$storageName/read", s"$storageName/write") - filterMetadataKeys(json) should equalIgnoreArrayOrder(expected) - response.status shouldEqual StatusCodes.OK - } + _ <- deltaClient.post[Json](s"/storages/$projectRef", payload, Coyote) { (_, response) => + response.status shouldEqual StatusCodes.Created + } + _ <- deltaClient.get[Json](s"/storages/$projectRef/nxv:$storId", Coyote) { (json, response) => + val expected = storageResponse(projectRef, storId, "resources/read", "files/write") + filterMetadataKeys(json) should equalIgnoreArrayOrder(expected) + response.status shouldEqual StatusCodes.OK + } + _ <- permissionDsl.addPermissions( + Permission(storName, "read"), + Permission(storName, "write") + ) + _ <- deltaClient.post[Json](s"/storages/$projectRef", payload2, Coyote) { (_, response) => + response.status shouldEqual StatusCodes.Created + } + _ <- deltaClient.get[Json](s"/storages/$projectRef/nxv:$storageId2", Coyote) { (json, response) => + val expected = storageResponse(projectRef, storageId2, storage2Read, storage2Write) + filterMetadataKeys(json) should equalIgnoreArrayOrder(expected) + response.status shouldEqual StatusCodes.OK + } } yield succeed } diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/RemoteStorageSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/RemoteStorageSpec.scala index 52abb4c436..966f06c015 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/RemoteStorageSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/RemoteStorageSpec.scala @@ -17,7 +17,7 @@ import org.scalatest.Assertion import scala.annotation.nowarn import scala.sys.process._ -class RemoteStorageSpec extends StorageSpec { +class RemoteStorageSpec extends StorageSpec with CopyFilesSpec { override def storageName: String = "external" @@ -60,59 +60,61 @@ class RemoteStorageSpec extends StorageSpec { ): _* ) - override def createStorages: IO[Assertion] = { - val payload = jsonContentOf( + override def createStorages(projectRef: String, storId: String, storName: String): IO[Assertion] = { + val payload = jsonContentOf( "kg/storages/remote-disk.json", "endpoint" -> externalEndpoint, "read" -> "resources/read", "write" -> "files/write", "folder" -> remoteFolder, - "id" -> storageId + "id" -> storId ) - - val payload2 = jsonContentOf( + val storageId2 = s"${storId}2" + val storage2Read = s"$storName/read" + val storage2Write = s"$storName/write" + val payload2 = jsonContentOf( "kg/storages/remote-disk.json", "endpoint" -> externalEndpoint, - "read" -> s"$storageName/read", - "write" -> s"$storageName/write", + "read" -> storage2Read, + "write" -> storage2Write, "folder" -> remoteFolder, - "id" -> s"${storageId}2" + "id" -> storageId2 ) for { - _ <- deltaClient.post[Json](s"/storages/$projectRef", payload, Coyote) { (json, response) => - if (response.status != StatusCodes.Created) { - fail(s"Unexpected status '${response.status}', response:\n${json.spaces2}") - } else succeed - } - _ <- deltaClient.get[Json](s"/storages/$projectRef/nxv:$storageId", Coyote) { (json, response) => - val expected = storageResponse(projectRef, storageId, "resources/read", "files/write") - filterMetadataKeys(json) should equalIgnoreArrayOrder(expected) - response.status shouldEqual StatusCodes.OK - } - _ <- deltaClient.get[Json](s"/storages/$projectRef/nxv:$storageId/source", Coyote) { (json, response) => - response.status shouldEqual StatusCodes.OK - val expected = jsonContentOf( - "kg/storages/storage-source.json", - "folder" -> remoteFolder, - "storageBase" -> externalEndpoint - ) - filterKey("credentials")(json) should equalIgnoreArrayOrder(expected) - - } - _ <- permissionDsl.addPermissions( - Permission(storageName, "read"), - Permission(storageName, "write") - ) - _ <- deltaClient.post[Json](s"/storages/$projectRef", payload2, Coyote) { (_, response) => - response.status shouldEqual StatusCodes.Created - } - storageId2 = s"${storageId}2" - _ <- deltaClient.get[Json](s"/storages/$projectRef/nxv:$storageId2", Coyote) { (json, response) => - val expected = storageResponse(projectRef, storageId2, s"$storageName/read", s"$storageName/write") - filterMetadataKeys(json) should equalIgnoreArrayOrder(expected) - response.status shouldEqual StatusCodes.OK - } + _ <- deltaClient.post[Json](s"/storages/$projectRef", payload, Coyote) { (json, response) => + if (response.status != StatusCodes.Created) { + fail(s"Unexpected status '${response.status}', response:\n${json.spaces2}") + } else succeed + } + _ <- deltaClient.get[Json](s"/storages/$projectRef/nxv:$storId", Coyote) { (json, response) => + val expected = storageResponse(projectRef, storId, "resources/read", "files/write") + filterMetadataKeys(json) should equalIgnoreArrayOrder(expected) + response.status shouldEqual StatusCodes.OK + } + _ <- deltaClient.get[Json](s"/storages/$projectRef/nxv:$storId/source", Coyote) { (json, response) => + response.status shouldEqual StatusCodes.OK + val expected = jsonContentOf( + "kg/storages/storage-source.json", + "folder" -> remoteFolder, + "storageBase" -> externalEndpoint, + "id" -> storId + ) + filterKey("credentials")(json) should equalIgnoreArrayOrder(expected) + + } + _ <- permissionDsl.addPermissions( + Permission(storName, "read"), + Permission(storName, "write") + ) + _ <- deltaClient.post[Json](s"/storages/$projectRef", payload2, Coyote) { (_, response) => + response.status shouldEqual StatusCodes.Created + } + _ <- deltaClient.get[Json](s"/storages/$projectRef/nxv:$storageId2", Coyote) { (json, response) => + val expected = storageResponse(projectRef, storageId2, storage2Read, storage2Write) + filterMetadataKeys(json) should equalIgnoreArrayOrder(expected) + response.status shouldEqual StatusCodes.OK + } } yield succeed } diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/S3StorageSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/S3StorageSpec.scala index 51af5b062c..00739d115c 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/S3StorageSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/S3StorageSpec.scala @@ -82,39 +82,39 @@ class S3StorageSpec extends StorageSpec { ): _* ) - override def createStorages: IO[Assertion] = { + override def createStorages(projectRef: String, storId: String, storName: String): IO[Assertion] = { val payload = jsonContentOf( "kg/storages/s3.json", - "storageId" -> s"https://bluebrain.github.io/nexus/vocabulary/$storageId", + "storageId" -> s"https://bluebrain.github.io/nexus/vocabulary/$storId", "bucket" -> bucket, "endpoint" -> s3Endpoint ) val payload2 = jsonContentOf( "kg/storages/s3.json", - "storageId" -> s"https://bluebrain.github.io/nexus/vocabulary/${storageId}2", + "storageId" -> s"https://bluebrain.github.io/nexus/vocabulary/${storId}2", "bucket" -> bucket, "endpoint" -> s3Endpoint ) deepMerge Json.obj( "region" -> Json.fromString("eu-west-2"), - "readPermission" -> Json.fromString(s"$storageName/read"), - "writePermission" -> Json.fromString(s"$storageName/write") + "readPermission" -> Json.fromString(s"$storName/read"), + "writePermission" -> Json.fromString(s"$storName/write") ) for { _ <- deltaClient.post[Json](s"/storages/$projectRef", payload, Coyote) { (_, response) => response.status shouldEqual StatusCodes.Created } - _ <- deltaClient.get[Json](s"/storages/$projectRef/nxv:$storageId", Coyote) { (json, response) => - val expected = storageResponse(projectRef, storageId, "resources/read", "files/write") + _ <- deltaClient.get[Json](s"/storages/$projectRef/nxv:$storId", Coyote) { (json, response) => + val expected = storageResponse(projectRef, storId, "resources/read", "files/write") filterMetadataKeys(json) should equalIgnoreArrayOrder(expected) response.status shouldEqual StatusCodes.OK } - _ <- permissionDsl.addPermissions(Permission(storageName, "read"), Permission(storageName, "write")) + _ <- permissionDsl.addPermissions(Permission(storName, "read"), Permission(storName, "write")) _ <- deltaClient.post[Json](s"/storages/$projectRef", payload2, Coyote) { (_, response) => response.status shouldEqual StatusCodes.Created } - storageId2 = s"${storageId}2" + storageId2 = s"${storId}2" _ <- deltaClient.get[Json](s"/storages/$projectRef/nxv:$storageId2", Coyote) { (json, response) => val expected = storageResponse(projectRef, storageId2, "s3/read", "s3/write") .deepMerge(Json.obj("region" -> Json.fromString("eu-west-2"))) diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/StorageSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/StorageSpec.scala index 50cde5ea45..0c5cf7e6c0 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/StorageSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/StorageSpec.scala @@ -19,6 +19,7 @@ import org.scalatest.Assertion import java.util.Base64 +final case class Input(fileId: String, filename: String, ct: ContentType, contents: String) abstract class StorageSpec extends BaseIntegrationSpec { val storageConfig: StorageConfig = load[StorageConfig](ConfigFactory.load(), "storage") @@ -39,7 +40,7 @@ abstract class StorageSpec extends BaseIntegrationSpec { def locationPrefix: Option[String] - def createStorages: IO[Assertion] + def createStorages(projectRef: String, storId: String, storName: String): IO[Assertion] protected def fileSelf(project: String, id: String): String = { val uri = Uri(s"${config.deltaUri}/files/$project") @@ -48,6 +49,18 @@ abstract class StorageSpec extends BaseIntegrationSpec { private[tests] val fileSelfPrefix = fileSelf(projectRef, attachmentPrefix) + val emptyFileContent = "" + val jsonFileContent = """{ "initial": ["is", "a", "test", "file"] }""" + val updatedJsonFileContent = """{ "updated": ["is", "a", "test", "file"] }""" + + val emptyTextFile = Input("empty", "empty", ContentTypes.`text/plain(UTF-8)`, emptyFileContent) + val jsonFileNoContentType = Input("attachment.json", "attachment.json", ContentTypes.NoContentType, jsonFileContent) + val updatedJsonFileWithContentType = + jsonFileNoContentType.copy(contents = updatedJsonFileContent, ct = ContentTypes.`application/json`) + val textFileNoContentType = Input("attachment2", "attachment2", ContentTypes.NoContentType, "text file") + val textFileWithContentType = + Input("attachment3", "attachment2", ContentTypes.`application/octet-stream`, "text file") + override def beforeAll(): Unit = { super.beforeAll() createProjects(Coyote, orgId, projId).accepted @@ -55,7 +68,7 @@ abstract class StorageSpec extends BaseIntegrationSpec { "Creating a storage" should { s"succeed for a $storageName storage" in { - createStorages + createStorages(projectRef, storageId, storageName) } "wait for storages to be indexed" in { @@ -70,16 +83,8 @@ abstract class StorageSpec extends BaseIntegrationSpec { "An empty file" should { - val emptyFileContent = "" - "be successfully uploaded" in { - deltaClient.uploadFile[Json]( - s"/files/$projectRef/empty?storage=nxv:$storageId", - emptyFileContent, - ContentTypes.`text/plain(UTF-8)`, - "empty", - Coyote - ) { expectCreated } + uploadFile(emptyTextFile, None)(expectCreated) } "be downloaded" in { @@ -91,19 +96,8 @@ abstract class StorageSpec extends BaseIntegrationSpec { "A json file" should { - val jsonFileContent = """{ "initial": ["is", "a", "test", "file"] }""" - val updatedJsonFileContent = """{ "updated": ["is", "a", "test", "file"] }""" - "be uploaded" in { - deltaClient.uploadFile[Json]( - s"/files/$projectRef/attachment.json?storage=nxv:$storageId", - jsonFileContent, - ContentTypes.NoContentType, - "attachment.json", - Coyote - ) { - expectCreated - } + uploadFile(jsonFileNoContentType, None)(expectCreated) } "be downloaded" in { @@ -119,15 +113,7 @@ abstract class StorageSpec extends BaseIntegrationSpec { } "be updated" in { - deltaClient.uploadFile[Json]( - s"/files/$projectRef/attachment.json?storage=nxv:$storageId&rev=1", - updatedJsonFileContent, - ContentTypes.`application/json`, - "attachment.json", - Coyote - ) { - expectOk - } + uploadFile(updatedJsonFileWithContentType, Some(1))(expectOk) } "download the updated file" in { @@ -184,23 +170,17 @@ abstract class StorageSpec extends BaseIntegrationSpec { "A file without extension" should { - val textFileContent = "text file" - "be uploaded" in { - deltaClient.uploadFile[Json]( - s"/files/$projectRef/attachment2?storage=nxv:$storageId", - textFileContent, - ContentTypes.NoContentType, - "attachment2", - Coyote - ) { - expectCreated - } + uploadFile(textFileNoContentType, None)(expectCreated) } "be downloaded" in { deltaClient.get[ByteString](s"/files/$projectRef/attachment:attachment2", Coyote, acceptAll) { - expectDownload("attachment2", ContentTypes.`application/octet-stream`, textFileContent) + expectDownload( + textFileNoContentType.filename, + ContentTypes.`application/octet-stream`, + textFileNoContentType.contents + ) } } } @@ -438,12 +418,31 @@ abstract class StorageSpec extends BaseIntegrationSpec { } + def uploadFile(fileInput: Input, rev: Option[Int]): ((Json, HttpResponse) => Assertion) => IO[Assertion] = + uploadFileToProjectStorage(fileInput, projectRef, storageId, rev) + + def uploadFileToProjectStorage( + fileInput: Input, + projRef: String, + storage: String, + rev: Option[Int] + ): ((Json, HttpResponse) => Assertion) => IO[Assertion] = { + val revString = rev.map(r => s"&rev=$r").getOrElse("") + deltaClient.uploadFile[Json]( + s"/files/$projRef/${fileInput.fileId}?storage=nxv:$storage$revString", + fileInput.contents, + fileInput.ct, + fileInput.filename, + Coyote + ) + } + private def attachmentString(filename: String): String = { val encodedFilename = new String(Base64.getEncoder.encode(filename.getBytes(Charsets.UTF_8))) s"=?UTF-8?B?$encodedFilename?=" } - private def expectDownload( + protected def expectDownload( expectedFilename: String, expectedContentType: ContentType, expectedContent: String,