Skip to content

Commit

Permalink
Fix
Browse files Browse the repository at this point in the history
  • Loading branch information
dantb committed Nov 28, 2023
1 parent f34c2f2 commit 755b50b
Show file tree
Hide file tree
Showing 14 changed files with 57 additions and 57 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import akka.actor.typed.scaladsl.adapter._
import akka.actor.{ActorSystem => ActorSystemClassic}
import akka.http.scaladsl.Http
import akka.http.scaladsl.server.{ExceptionHandler, RejectionHandler, Route, RouteResult}
import cats.effect.{Blocker, ExitCode, IO, IOApp, Resource}
import cats.effect.{ExitCode, IO, IOApp, Resource}
import cats.syntax.all._
import ch.epfl.bluebrain.nexus.delta.config.{AppConfig, BuildInfo}
import ch.epfl.bluebrain.nexus.delta.kernel.Logger
Expand Down Expand Up @@ -53,8 +53,7 @@ object Main extends IOApp {
(cfg, config, cl, pluginDefs) <- Resource.eval(loadPluginsAndConfig(loaderConfig))
_ <- Resource.eval(KamonMonitoring.initialize(config))
modules = DeltaModule(cfg, config, cl)
fileOperationBlocker <- Blocker[IO]
(plugins, locator) <- WiringInitializer(modules, pluginDefs, fileOperationBlocker)
(plugins, locator) <- WiringInitializer(modules, pluginDefs)
_ <- bootstrap(locator, plugins)
} yield locator

Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package ch.epfl.bluebrain.nexus.delta.plugin

import akka.http.scaladsl.testkit.ScalatestRouteTest
import cats.effect.{Blocker, IO}
import cats.effect.IO
import ch.epfl.bluebrain.nexus.delta.plugin.PluginsLoader.PluginLoaderConfig
import ch.epfl.bluebrain.nexus.delta.sdk.PriorityRoute
import ch.epfl.bluebrain.nexus.delta.sdk.model.BaseUri
Expand All @@ -19,9 +19,8 @@ class PluginLoaderSpec extends CatsEffectSpec with ScalatestRouteTest {
"A PluginLoader" should {
val config = PluginLoaderConfig("../plugins/test-plugin/target")
"load plugins from .jar in a directory" in {
val blocker = Blocker[IO].allocated.unsafeRunSync()._1
val (_, pluginsDef) = PluginsLoader(config).load.accepted
WiringInitializer(serviceModule, pluginsDef, blocker).use { case (_, locator) =>
WiringInitializer(serviceModule, pluginsDef).use { case (_, locator) =>
IO.delay {
val route = locator.get[Set[PriorityRoute]].head
pluginsDef.head.priority shouldEqual 10
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.files
import akka.actor.typed.ActorSystem
import akka.actor.{ActorSystem => ClassicActorSystem}
import akka.http.scaladsl.model.ContentTypes.`application/octet-stream`
import akka.http.scaladsl.model.{ContentType, HttpEntity, Uri}
import akka.http.scaladsl.model.{BodyPartEntity, ContentType, HttpEntity, Uri}
import cats.effect.{Clock, IO}
import cats.syntax.all._
import ch.epfl.bluebrain.nexus.delta.kernel.cache.LocalCache
Expand Down Expand Up @@ -210,8 +210,10 @@ final class Files(
file <- fetchSourceFile(sourceId)
(pc, destStorageRef, destStorage) <- fetchDestinationStorage(dest)
_ <- validateStorageTypeForCopy(file.storageType, destStorage)
space <- fetchStorageAvailableSpace(destStorage)
_ <- IO.raiseUnless(space.exists(_ < file.attributes.bytes))(FileTooLarge(destStorage.storageValue.maxFileSize, space))
space <- fetchStorageAvailableSpace(destStorage)
_ <- IO.raiseUnless(space.exists(_ < file.attributes.bytes))(
FileTooLarge(destStorage.storageValue.maxFileSize, space)
)
iri <- dest.fileId.fold(generateId(pc))(FileId(_, dest.project).expandIri(fetchContext.onCreate).map(_._1))
destinationDesc <- FileDescription(dest.filename.getOrElse(file.attributes.filename), file.attributes.mediaType)
attributes <- CopyFile(destStorage, remoteDiskStorageClient).apply(file.attributes, destinationDesc).adaptError {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.{D
import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote._
import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.remote.client.RemoteDiskStorageClient
import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.s3.{S3StorageFetchFile, S3StorageLinkFile, S3StorageSaveFile}
import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.{CopyFile, FetchAttributes, FetchFile, LinkFile, SaveFile}
import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations._
import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.{contexts, Storages}
import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri
import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.ContextValue
Expand Down Expand Up @@ -89,7 +89,7 @@ object Storage {
def saveFile(implicit as: ActorSystem): SaveFile =
new DiskStorageSaveFile(this)

def copyFile(implicit b: Blocker, cs: ContextShift[IO]): CopyFile = new DiskStorageCopyFile(this)
def copyFile: CopyFile = new DiskStorageCopyFile(this)
}

/**
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations

import cats.effect.{Blocker, ContextShift, IO}
import cats.effect.IO
import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes, FileDescription}
import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.{Storage, StorageType}
import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.StorageFileRejection.CopyFileRejection
Expand All @@ -12,7 +12,7 @@ trait CopyFile {

object CopyFile {

def apply(storage: Storage, client: RemoteDiskStorageClient)(implicit b: Blocker, cs: ContextShift[IO]): CopyFile =
def apply(storage: Storage, client: RemoteDiskStorageClient): CopyFile =
storage match {
case storage: Storage.DiskStorage => storage.copyFile
case storage: Storage.S3Storage => unsupported(storage.tpe)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,21 +1,23 @@
package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk

import akka.http.scaladsl.model.Uri
import cats.effect.{Blocker, ContextShift, IO}
import cats.effect.IO
import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.{FileAttributes, FileDescription}
import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.Storage.DiskStorage
import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.CopyFile
import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk.DiskStorageSaveFile.initLocation

import java.net.URI
import java.nio.file.{Paths, StandardCopyOption}
import scala.annotation.nowarn

class DiskStorageCopyFile(storage: DiskStorage)(implicit blocker: Blocker, cs: ContextShift[IO]) extends CopyFile {
class DiskStorageCopyFile(storage: DiskStorage) extends CopyFile {
@nowarn
override def apply(source: FileAttributes, dest: FileDescription): IO[FileAttributes] = {
val sourcePath = Paths.get(URI.create(s"file://${source.location.path}"))
for {
(destPath, destRelativePath) <- initLocation(storage.project, storage.value, dest.uuid, dest.filename)
_ <- fs2.io.file.copy[IO](blocker, sourcePath, destPath, Seq(StandardCopyOption.COPY_ATTRIBUTES))
_ <- fs2.io.file.copy[IO](sourcePath, destPath, Seq(StandardCopyOption.COPY_ATTRIBUTES))
} yield FileAttributes(
uuid = dest.uuid,
location = Uri(destPath.toUri.toString),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.operations.disk

import akka.actor.ActorSystem
import akka.http.scaladsl.model.{BodyPartEntity, Uri}
import akka.stream.{IOOperationIncompleteException, IOResult}
import akka.stream.IOOperationIncompleteException
import akka.stream.scaladsl.FileIO
import cats.effect.IO
import cats.implicits._
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,16 +16,16 @@ class RemoteDiskStorageCopyFile(
def apply(source: FileAttributes, description: FileDescription): IO[FileAttributes] = {
val destinationPath = Uri.Path(intermediateFolders(storage.project, description.uuid, description.filename))
client.copyFile(storage.value.folder, source.location.path, destinationPath)(storage.value.endpoint).as {
FileAttributes(
uuid = description.uuid,
location = source.location, // TODO what's the destination absolute path?
path = destinationPath,
filename = description.filename,
mediaType = description.mediaType,
bytes = source.bytes,
digest = source.digest,
origin = source.origin
)
FileAttributes(
uuid = description.uuid,
location = source.location, // TODO what's the destination absolute path?
path = destinationPath,
filename = description.filename,
mediaType = description.mediaType,
bytes = source.bytes,
digest = source.digest,
origin = source.origin
)
}

}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -176,34 +176,34 @@ final class RemoteDiskStorageClient(client: HttpClient, getAuthToken: AuthTokenP
}

/**
* Moves a path from the provided ''sourceRelativePath'' to ''destRelativePath'' inside the nexus folder.
*
* @param bucket
* the storage bucket name
* @param sourceRelativePath
* the source relative path location
* @param destRelativePath
* the destination relative path location inside the nexus folder
*/
* Moves a path from the provided ''sourceRelativePath'' to ''destRelativePath'' inside the nexus folder.
*
* @param bucket
* the storage bucket name
* @param sourceRelativePath
* the source relative path location
* @param destRelativePath
* the destination relative path location inside the nexus folder
*/
def copyFile(
bucket: Label,
sourceRelativePath: Path,
destRelativePath: Path
)(implicit baseUri: BaseUri): IO[Unit] = {
bucket: Label,
sourceRelativePath: Path,
destRelativePath: Path
)(implicit baseUri: BaseUri): IO[Unit] = {
getAuthToken(credentials).flatMap { authToken =>
val endpoint = baseUri.endpoint / "buckets" / bucket.value / "files" / destRelativePath
val payload = Json.obj("source" -> sourceRelativePath.toString.asJson)
val payload = Json.obj("source" -> sourceRelativePath.toString.asJson)
client
.discardBytes(Post(endpoint, payload).withCredentials(authToken), ())
.adaptError {
// TODO update error
case error@HttpClientStatusError(_, `NotFound`, _) if !bucketNotFoundType(error) =>
// TODO update error
case error @ HttpClientStatusError(_, `NotFound`, _) if !bucketNotFoundType(error) =>
MoveFileRejection.FileNotFound(sourceRelativePath.toString)
case error@HttpClientStatusError(_, `BadRequest`, _) if pathContainsLinksType(error) =>
case error @ HttpClientStatusError(_, `BadRequest`, _) if pathContainsLinksType(error) =>
MoveFileRejection.PathContainsLinks(destRelativePath.toString)
case HttpClientStatusError(_, `Conflict`, _) =>
case HttpClientStatusError(_, `Conflict`, _) =>
MoveFileRejection.ResourceAlreadyExists(destRelativePath.toString)
case error: HttpClientError =>
case error: HttpClientError =>
UnexpectedMoveError(sourceRelativePath.toString, destRelativePath.toString, error.asString)
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.files

import akka.http.scaladsl.model.ContentTypes.`text/plain(UTF-8)`
import akka.http.scaladsl.model.{HttpEntity, MessageEntity, Multipart, Uri}
import cats.effect.unsafe.implicits.global
import cats.effect.{IO, Ref}
import ch.epfl.bluebrain.nexus.delta.kernel.utils.{UUIDF, UrlUtils}
import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest.ComputedDigest
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import akka.actor.{typed, ActorSystem}
import akka.http.scaladsl.model.ContentTypes.`text/plain(UTF-8)`
import akka.http.scaladsl.model.Uri
import akka.testkit.TestKit
import cats.effect.{Blocker, IO}
import cats.effect.IO
import ch.epfl.bluebrain.nexus.delta.kernel.http.MediaTypeDetectorConfig
import ch.epfl.bluebrain.nexus.delta.plugins.storage.RemoteContextResolutionFixture
import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest.NotComputedDigest
Expand Down Expand Up @@ -63,7 +63,7 @@ class FilesSpec(docker: RemoteStorageDocker)

"The Files operations bundle" when {
implicit val typedSystem: typed.ActorSystem[Nothing] = system.toTyped
implicit val httpClient: HttpClient = HttpClient()(httpClientConfig, system, timer, contextShift)
implicit val httpClient: HttpClient = HttpClient()(httpClientConfig, system)
implicit val caller: Caller = Caller(bob, Set(bob, Group("mygroup", realm), Authenticated(realm)))
implicit val authTokenProvider: AuthTokenProvider = AuthTokenProvider.anonymousForTest
val remoteDiskStorageClient = new RemoteDiskStorageClient(httpClient, authTokenProvider, Credentials.Anonymous)
Expand Down Expand Up @@ -121,8 +121,6 @@ class FilesSpec(docker: RemoteStorageDocker)
clock
).accepted

implicit val blocker = Blocker[IO].allocated.unsafeRunSync()._1

lazy val files: Files = Files(
fetchContext.mapRejection(FileRejection.ProjectContextRejection),
aclCheck,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import akka.http.scaladsl.model.MediaTypes.`text/html`
import akka.http.scaladsl.model.headers.{Accept, Location, OAuth2BearerToken, RawHeader}
import akka.http.scaladsl.model.{StatusCodes, Uri}
import akka.http.scaladsl.server.Route
import cats.effect.{Blocker, IO}
import cats.effect.IO
import ch.epfl.bluebrain.nexus.delta.kernel.http.MediaTypeDetectorConfig
import ch.epfl.bluebrain.nexus.delta.kernel.utils.ClasspathResourceLoader
import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.model.Digest.ComputedDigest
Expand All @@ -26,7 +26,7 @@ import ch.epfl.bluebrain.nexus.delta.sdk.acls.AclSimpleCheck
import ch.epfl.bluebrain.nexus.delta.sdk.acls.model.AclAddress
import ch.epfl.bluebrain.nexus.delta.sdk.auth.{AuthTokenProvider, Credentials}
import ch.epfl.bluebrain.nexus.delta.sdk.directives.DeltaSchemeDirectives
import ch.epfl.bluebrain.nexus.delta.sdk.http.HttpClient
import ch.epfl.bluebrain.nexus.delta.sdk.http.{HttpClient, HttpClientConfig}
import ch.epfl.bluebrain.nexus.delta.sdk.identities.model.{Caller, ServiceAccount}
import ch.epfl.bluebrain.nexus.delta.sdk.identities.{Identities, IdentitiesDummy}
import ch.epfl.bluebrain.nexus.delta.sdk.implicits._
Expand Down Expand Up @@ -143,7 +143,7 @@ class FilesRoutesSpec
DeltaSchemeDirectives(
fetchContext,
ioFromMap(uuid -> projectRef.organization, uuidOrg2 -> projectRefOrg2.organization),
ioFromMap(uuid -> projectRef, uuidOrg2 -> projectRefOrg2)
ioFromMap(uuid -> projectRef, uuidOrg2 -> projectRefOrg2)
)
private lazy val routes = routesWithIdentities(identities)
private def routesWithIdentities(identities: Identities) =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package ch.epfl.bluebrain.nexus.tests.kg
import akka.http.scaladsl.model._
import akka.util.ByteString
import cats.effect.IO
import cats.implicits.catsSyntaxFlatMapOps
import ch.epfl.bluebrain.nexus.delta.kernel.utils.UrlUtils
import ch.epfl.bluebrain.nexus.tests.HttpClient._
import ch.epfl.bluebrain.nexus.tests.Identity.storages.Coyote
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -846,7 +846,7 @@ class SearchConfigIndexingSpec extends BaseIntegrationSpec {
}

"aggregate presynaptic brain regions" in {
val query = jsonContentOf("kg/search/synapse-agg.json")
val query = jsonContentOf("kg/search/synapse-agg.json")
val preSynapticBrainRegionAgg =
json"""{
"preSynapticBrainRegions" : {
Expand Down Expand Up @@ -877,10 +877,10 @@ class SearchConfigIndexingSpec extends BaseIntegrationSpec {
* Defines an ES query that searches for the document with the provided id and limits the resulting source to just
* the requested field
*/
private def queryField(id: String, field: String) =
private def queryField(id: String, field: String) =
jsonContentOf("kg/search/id-query-single-field.json", "id" -> id, "field" -> field)

private def queryDocument(id: String) =
private def queryDocument(id: String) =
jsonContentOf("kg/search/id-query.json", "id" -> id)

private def aggregationIn(json: Json): Option[Json] =
Expand Down

0 comments on commit 755b50b

Please sign in to comment.