Add encryption middleware to file storage

LICENSE

@@ -4,6 +4,8 @@ Original work (as in https://github.com/pityka/mybiotools/tree/master/Tasks):
 Copyright (c) 2015 ECOLE POLYTECHNIQUE FEDERALE DE LAUSANNE, Switzerland, Group Fellay
 
 Modified work (this repository): Copyright (c) 2016 Istvan Bartha
+With the following exceptions:
+- Aes.scala is distributed under MIT License, Copyright (c) 2023 Jakub Wojnowski. See header.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

core/src/main/scala/tasks/TaskSystemComponents.scala

@@ -280,14 +280,29 @@ object TaskSystemComponents {
           }
         }
 
-      fileStore.flatMap { fileStore =>
-        fileStore match {
-          case fs: ManagedFileStorage if config.proxyStorage =>
-            proxyFileStorageHttpServer(fs).map(_ => fs)
-
+      fileStore
+        .flatMap {
+          case fs: ManagedFileStorage
+              if config.storageEncryptionKey.isDefined =>
+            Resource.make(
+              IO(
+                new EncryptedManagedFileStorage(
+                  fs,
+                  config.storageEncryptionKey.get
+                )
+              )
+            )(e => IO(e.destroyKey()))
           case fs: ManagedFileStorage => Resource.pure(fs)
+
+        }
+        .flatMap { fileStore =>
+          fileStore match {
+            case fs: ManagedFileStorage if config.proxyStorage =>
+              proxyFileStorageHttpServer(fs).map(_ => fs)
+
+            case fs: ManagedFileStorage => Resource.pure(fs)
+          }
         }
-      }
 
     }
 

core/src/main/scala/tasks/fileservice/EncryptedManagedFileStorage.scala

@@ -0,0 +1,114 @@
+/*
+ * The MIT License
+ *
+ * Copyright (c) 2015 ECOLE POLYTECHNIQUE FEDERALE DE LAUSANNE, Switzerland,
+ * Group Fellay
+ * Modified work, Copyright (c) 2016 Istvan Bartha
+
+
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the Software
+ * is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package tasks.fileservice
+
+import tasks.util._
+import tasks.util.config._
+import tasks.util.eq._
+
+import java.io.File
+
+import cats.effect.kernel.Resource
+import cats.effect.IO
+import fs2.{Stream, Pipe}
+import software.amazon.awssdk.services.s3.model.HeadObjectResponse
+import tasks.fileservice._
+import cats.effect.std.SecureRandom
+
+class EncryptedManagedFileStorage(
+    parent: ManagedFileStorage,
+    keyHex: String
+) extends ManagedFileStorage {
+
+  override def toString = s"Encrypted($parent)"
+
+  private val key = Aes
+    .keyFromHex(keyHex)
+    .getOrElse(
+      throw new RuntimeException(
+        "Not proper key. Key must be 32 bytes encoded as a hex string (00-ff)."
+      )
+    )
+
+  def destroyKey() = key.destroy()
+
+  def sharedFolder(prefix: Seq[String]): IO[Option[File]] = IO.pure(None)
+
+  def contains(
+      path: ManagedFilePath,
+      retrieveSizeAndHash: Boolean
+  ): IO[Option[SharedFile]] = parent.contains(path, retrieveSizeAndHash)
+
+  def contains(path: ManagedFilePath, size: Long, hash: Int): IO[Boolean] =
+    parent.contains(path, size, hash)
+
+  def sink(
+      path: ProposedManagedFilePath
+  ): Pipe[IO, Byte, (Long, Int, ManagedFilePath)] = { (in: Stream[IO, Byte]) =>
+    Stream.eval(SecureRandom.javaSecuritySecureRandom[IO]).flatMap {
+      implicit secureRandom =>
+        in.through(Aes.encrypt(key)).through(parent.sink(path))
+    }
+  }
+
+  def stream(
+      path: ManagedFilePath,
+      fromOffset: Long
+  ): Stream[IO, Byte] = {
+    parent.stream(path, 0L).through(Aes.decrypt(key)).drop(fromOffset)
+  }
+
+  def exportFile(path: ManagedFilePath): Resource[IO, File] =
+    parent.exportFile(path).flatMap { encrypted =>
+      Resource.make(IO {
+        val tmp = TempFile.createTempFile("")
+        fs2.io.file
+          .Files[IO]
+          .readAll(fs2.io.file.Path.fromNioPath(encrypted.toPath))
+          .through(Aes.decrypt(key))
+          .through(
+            fs2.io.file
+              .Files[IO]
+              .writeAll(fs2.io.file.Path.fromNioPath(tmp.toPath))
+          )
+          .compile
+          .drain
+          .map { _ =>
+            tmp
+          }
+      }.flatten)(f => IO.delay(f.delete))
+    }
+
+  def uri(mp: ManagedFilePath): IO[tasks.util.Uri] =
+    parent.uri(mp)
+
+  def delete(mp: ManagedFilePath, size: Long, hash: Int) =
+    parent.delete(mp, size, hash)
+
+}

core/src/main/scala/tasks/util/Aes.scala

@@ -0,0 +1,181 @@
+// MIT License
+
+// Copyright (c) 2023 Jakub Wojnowski
+
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+package tasks.util
+
+import fs2.Chunk
+import fs2.Pipe
+import fs2.Pull
+import fs2.RaiseThrowable
+import fs2.Stream
+
+import cats.effect.Sync
+import cats.effect.std.SecureRandom
+import cats.syntax.all._
+
+import scala.util.Try
+import scala.util.control.NonFatal
+
+import java.nio.ByteBuffer
+
+import javax.crypto.Cipher
+import javax.crypto.SecretKey
+import javax.crypto.spec.GCMParameterSpec
+import javax.crypto.spec.SecretKeySpec
+
+object Aes {
+
+  private val IntSizeInBytes = 4
+  private val IvLengthBytes = 12
+  private val AuthTagLengthBytes = 16
+  private val DefaultChunkSize = 4 * 1024 * 1024
+
+  private val transformation = "AES/GCM/NoPadding"
+  private val keyAlgorithm = "AES"
+
+  def generateKeyHexString(implicit
+      ev: SecureRandom[cats.effect.IO]
+  ): cats.effect.IO[String] = {
+    SecureRandom[cats.effect.IO]
+      .nextBytes(32) // 256 bits
+      .map(_.map("%02x".format(_)).mkString)
+  }
+
+  def decrypt[F[_]: Sync](key: SecretKey): Pipe[F, Byte, Byte] =
+    (stream: Stream[F, Byte]) =>
+      readFirstN(IntSizeInBytes, stream) { (chunkSizeBytes, remainingStream) =>
+        val chunkSize = bytesToChunkSize(chunkSizeBytes)
+
+        remainingStream
+          .chunkN(IvLengthBytes + chunkSize + AuthTagLengthBytes)
+          .flatMap { chunk =>
+            readFirstN(IvLengthBytes, Stream.chunk(chunk).covary[F]) {
+              (ivChunk, stream) =>
+                Stream
+                  .eval(createCipher(Cipher.DECRYPT_MODE, key, ivChunk.toArray))
+                  .flatMap { cipher =>
+                    stream.mapChunks(cipherChunk(cipher, _))
+                  }
+            }
+          }
+          .adaptError { case NonFatal(throwable) =>
+            Error.DecryptionError(throwable)
+          }
+
+      }
+
+  def encrypt[F[_]: Sync: SecureRandom](
+      key: SecretKey,
+      chunkSize: Int = DefaultChunkSize
+  ): Pipe[F, Byte, Byte] =
+    (stream: Stream[F, Byte]) =>
+      Stream.chunk(chunkSizeToBytes(chunkSize)) ++
+        stream
+          .chunkN(chunkSize)
+          .flatMap { chunk =>
+            Stream.eval(SecureRandom[F].nextBytes(IvLengthBytes)).flatMap {
+              ivBytes =>
+                Stream
+                  .eval(createCipher[F](Cipher.ENCRYPT_MODE, key, ivBytes))
+                  .flatMap { cipher =>
+                    Stream.chunk(Chunk.array(ivBytes)) ++ Stream
+                      .chunk(cipherChunk(cipher, chunk))
+                  }
+            }
+          }
+          .adaptError { case NonFatal(throwable) =>
+            Error.EncryptionError(throwable)
+          }
+
+  def keyFromHex(hexString: String): Option[SecretKey] =
+    Try {
+      new SecretKeySpec(
+        hexString.sliding(2, 2).toArray.map(Integer.parseInt(_, 16).toByte),
+        keyAlgorithm
+      )
+    }.toOption
+
+  private type Mode = Int
+
+  private def createCipher[F[_]: Sync](
+      mode: Mode,
+      key: SecretKey,
+      ivBytes: Array[Byte]
+  ): F[Cipher] =
+    Sync[F].delay {
+      val cipher = Cipher.getInstance(transformation)
+      cipher.init(
+        mode,
+        key,
+        new GCMParameterSpec(AuthTagLengthBytes * 8, ivBytes)
+      )
+      cipher
+    }
+
+  private def readFirstN[F[_]: RaiseThrowable, A, B](
+      n: Int,
+      stream: Stream[F, A]
+  )(
+      f: (Chunk[A], Stream[F, A]) => Stream[F, B]
+  ): Stream[F, B] =
+    stream.pull
+      .unconsN(n)
+      .flatMap {
+        case Some((chunk, stream)) =>
+          f(chunk, stream).pull.echo
+        case None =>
+          Pull.raiseError(Error.DataTooShort)
+      }
+      .stream
+
+  private def cipherChunk(cipher: Cipher, chunk: Chunk[Byte]): Chunk[Byte] = {
+    val inputBuffer = chunk.toByteBuffer
+    val outputBuffer =
+      ByteBuffer.allocate(cipher.getOutputSize(inputBuffer.remaining()))
+    cipher.doFinal(inputBuffer, outputBuffer)
+    Chunk.byteBuffer(outputBuffer.rewind())
+  }
+
+  private def chunkSizeToBytes(chunkSize: Int): Chunk[Byte] = {
+    val buffer = ByteBuffer.allocate(IntSizeInBytes)
+    buffer.putInt(chunkSize)
+    Chunk.byteBuffer(buffer.rewind())
+  }
+
+  private def bytesToChunkSize(bytes: Chunk[Byte]): Int =
+    bytes.toByteBuffer.getInt
+
+  sealed abstract class Error(message: String, cause: Option[Throwable] = None)
+      extends Exception(message, cause.orNull)
+      with Product
+      with Serializable
+
+  object Error {
+    case object DataTooShort extends Error("Data too short")
+
+    case class EncryptionError(cause: Throwable)
+        extends Error(s"Error during encryption: $cause", cause.some)
+
+    case class DecryptionError(cause: Throwable)
+        extends Error(s"Error during decryption: $cause", cause.some)
+  }
+
+}

core/src/main/scala/tasks/util/config/TasksConfig.scala

@@ -303,4 +303,9 @@ class TasksConfig(load: () => Config) {
   val connectToProxyFileServiceOnMain =
     raw.getBoolean("tasks.fileservice.connectToProxy")
 
+  val storageEncryptionKey =
+    if (raw.hasPath("tasks.fileservice.encryptionKey"))
+      Some(raw.getString("tasks.fileservice.encryptionKey"))
+    else None
+
 }