diff --git a/build.sbt b/build.sbt index 69b9f7ae..cdf87415 100644 --- a/build.sbt +++ b/build.sbt @@ -157,7 +157,7 @@ lazy val core = project "com.typesafe.akka" %% "akka-testkit" % akkaVersion % "test", "com.typesafe" % "config" % "1.4.2", "org.typelevel" %% "cats-effect" % "3.5.3", - "io.github.pityka" %% "selfpackage" % "2.1.0", + "io.github.pityka" %% "selfpackage" % "2.1.6", "org.scalatest" %% "scalatest" % "3.2.19" % "test", "com.outr" %% "scribe" % scribeVersion, "com.outr" %% "scribe-slf4j" % scribeVersion, @@ -203,7 +203,7 @@ lazy val kubernetes = project name := "tasks-kubernetes", libraryDependencies ++= Seq( "com.goyeau" %% "kubernetes-client" % "0.11.0", - "io.github.pityka" %% "selfpackage-jib" % "2.1.3", + "io.github.pityka" %% "selfpackage-jib" % "2.1.6", ) ++ akkaProvided ) diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index a7f0ec46..2ccccccd 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -348,6 +348,8 @@ tasks.ui.app.port = "28881" tasks.kubernetes.image = "" +tasks.kubernetes.imageApplicationSubPath = "/tasksapp" + tasks.kubernetes.namespace = "default" # list groups of 5 values diff --git a/core/src/main/scala/tasks/elastic/Deployment.scala b/core/src/main/scala/tasks/elastic/Deployment.scala index 4e641682..bec8e6d3 100644 --- a/core/src/main/scala/tasks/elastic/Deployment.scala +++ b/core/src/main/scala/tasks/elastic/Deployment.scala @@ -67,7 +67,7 @@ object Deployment { s"-Dhosts.gpusAsCommaString=${gpus.map(_.toString).mkString(",")}" else "" - val hostImageString = if (image.isDefined) s"-Dhosts.image=$image" else "" + val hostImageString = if (image.isDefined) s"-Dhosts.image=${image.get}" else "" val edited = s"./$packageFileName -J-Xmx{RAM}M -Dtasks.elastic.engine={GRID} {EXTRA} -Dhosts.master={MASTER} -Dhosts.app=false -Dtasks.fileservice.storageURI={STORAGE} -Dhosts.numCPU=$cpu -Dhosts.RAM=$memory -Dhosts.scratch=$scratch $gpuString $hostnameString $hostImageString" diff --git a/core/src/main/scala/tasks/util/config/TasksConfig.scala b/core/src/main/scala/tasks/util/config/TasksConfig.scala index 8c2ee03d..b6ad122e 100644 --- a/core/src/main/scala/tasks/util/config/TasksConfig.scala +++ b/core/src/main/scala/tasks/util/config/TasksConfig.scala @@ -250,6 +250,7 @@ class TasksConfig(load: () => Config) { val appUIServerPort = raw.getInt("tasks.ui.app.port") def kubernetesImageName = raw.getString("tasks.kubernetes.image") + val kubernetesImageApplicationSubPath = raw.getString("tasks.kubernetes.imageApplicationSubPath") def kubernetesHostNameOrIPEnvVar = raw.getString("tasks.kubernetes.hostnameOrIPEnvVar") diff --git a/kubernetes-test/src/main/scala/KubernetesTest.scala b/kubernetes-test/src/main/scala/KubernetesTest.scala index 1dd490eb..9a76b23b 100644 --- a/kubernetes-test/src/main/scala/KubernetesTest.scala +++ b/kubernetes-test/src/main/scala/KubernetesTest.scala @@ -57,7 +57,6 @@ object KubernetesTest extends IOApp { IO(1) } - // tasks.worker-main-class = "tasks.KubernetesTestSlave" val testConfig2 = { val tmp = tasks.util.TempFile.createTempFile(".temp") tmp.delete @@ -69,7 +68,7 @@ object KubernetesTest extends IOApp { tasks.elastic.queueCheckInterval = 3 seconds tasks.addShutdownHook = false tasks.failuredetector.acceptable-heartbeat-pause = 10 s - tasks.kubernetes.image = "eclipse-temurin-unzip" + tasks.kubernetes.image = "eclipse-temurin:17.0.13_11-jre-ubi9-minimal" tasks.kubernetes.image-pull-policy = "IfNotPresent" tasks.kubernetes.podSpec = { diff --git a/kubernetes/src/main/scala/tasks/elastic/kubernetes/Bootstrap.scala b/kubernetes/src/main/scala/tasks/elastic/kubernetes/Bootstrap.scala index 36ecc99a..0b1c9fce 100644 --- a/kubernetes/src/main/scala/tasks/elastic/kubernetes/Bootstrap.scala +++ b/kubernetes/src/main/scala/tasks/elastic/kubernetes/Bootstrap.scala @@ -117,9 +117,12 @@ object Bootstrap { scribe.info("No MY_POD_IP env found. Create pod of master.") k8sClientResource .flatMap { k8s => + val pathOfEntrypointInBootstrapContainer = + tconfig.kubernetesImageApplicationSubPath val container = selfpackage.jib.containerize( out = addScribe(containerizer), - mainClassNameArg = Some(mainClassName) + mainClassNameArg = Some(mainClassName), + pathInContainer = pathOfEntrypointInBootstrapContainer ) scribe.info( @@ -133,7 +136,7 @@ object Bootstrap { val kubeCPURequest = userCPURequest + tconfig.kubernetesCpuExtra val kubeRamRequest = userRamRequest + tconfig.kubernetesRamExtra - val podName = ("main-" + KubernetesHelpers.newName).take(47) + val podName = ("tasks-app-" + KubernetesHelpers.newName).take(47) val imageName = container.getTargetImage().toString @@ -145,6 +148,7 @@ object Bootstrap { } .getOrElse(PodSpec(containers = Nil)) + val containerName = "tasks-app" val resource = Pod( metadata = Some( ObjectMeta( @@ -160,8 +164,13 @@ object Bootstrap { containers = List( Container( image = Some(imageName), - command = Some(List("bash", "/app/entrypoint.sh")), - name = "tasks-master", + command = Some( + List( + "bash", + s"$pathOfEntrypointInBootstrapContainer/entrypoint.sh" + ) + ), + name = containerName, imagePullPolicy = Some(tconfig.kubernetesImagePullPolicy), env = Some( List( @@ -237,23 +246,36 @@ object Bootstrap { } val podIsRunning = phaseStream - .takeWhile(either => !either.exists(_.exists(_ == "Running"))) + .takeThrough(either => + either match { + case Left(error) => + scribe.error(error) + false + case Right(None) => + scribe.error("No phase in pod status") + false + case Right(Some(phase)) => + phase != "Running" && phase != "Failed" + } + ) .evalTap(phase => IO { scribe.info(s"$podName in phase $phase") } ) .compile - .drain - .flatTap(_ => - IO { scribe.info(s"$podName reached Running phase. ") } + .last + .map(option => + option.exists(_.exists(_.exists(_ == "Running"))) ) fs2.Stream .eval(podIsRunning) - .flatMap(_ => - k8s.pods - .namespace(tconfig.kubernetesNamespace) - .log(podName, Some("tasks-master"), follow = true) - .flatMap(response => response.bodyText) + .flatMap(podIsRunning => + if (podIsRunning) + k8s.pods + .namespace(tconfig.kubernetesNamespace) + .log(podName, Some(containerName), follow = true) + .flatMap(response => response.bodyText) + else fs2.Stream.empty ) } else { scribe.error("Failed pod creation") @@ -280,7 +302,10 @@ object Bootstrap { .flatMap(_ => k8s.pods.namespace(tconfig.kubernetesNamespace).delete(podName) ) - .map(_ => None) + .map(deletionStatus => { + scribe.info(s"Deletion status $deletionStatus") + None + }) ) } diff --git a/project/metals.sbt b/project/metals.sbt index be029c0a..dc3c19f5 100644 --- a/project/metals.sbt +++ b/project/metals.sbt @@ -3,6 +3,6 @@ // This file enables sbt-bloop to create bloop config files. -addSbtPlugin("ch.epfl.scala" % "sbt-bloop" % "2.0.3") +addSbtPlugin("ch.epfl.scala" % "sbt-bloop" % "2.0.8") // format: on