Let kubernetes test run on a ubi-minimal image

pityka · Feb 14, 2025 · c77ff03 · c77ff03
1 parent 893e83c
commit c77ff03
Show file tree

Hide file tree

Showing 7 changed files with 47 additions and 20 deletions.
diff --git a/build.sbt b/build.sbt
@@ -157,7 +157,7 @@ lazy val core = project
       "com.typesafe.akka" %% "akka-testkit" % akkaVersion % "test",
       "com.typesafe" % "config" % "1.4.2",
       "org.typelevel" %% "cats-effect" % "3.5.3",
-      "io.github.pityka" %% "selfpackage" % "2.1.0",
+      "io.github.pityka" %% "selfpackage" % "2.1.6",
       "org.scalatest" %% "scalatest" % "3.2.19" % "test",
       "com.outr" %% "scribe" % scribeVersion,
       "com.outr" %% "scribe-slf4j" % scribeVersion,
@@ -203,7 +203,7 @@ lazy val kubernetes = project
     name := "tasks-kubernetes",
     libraryDependencies ++= Seq(
       "com.goyeau" %% "kubernetes-client" % "0.11.0",
-      "io.github.pityka" %% "selfpackage-jib" % "2.1.3",
+      "io.github.pityka" %% "selfpackage-jib" % "2.1.6",
 
     ) ++ akkaProvided
   )

diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf
@@ -348,6 +348,8 @@ tasks.ui.app.port = "28881"
 
 tasks.kubernetes.image = ""
 
+tasks.kubernetes.imageApplicationSubPath = "/tasksapp"
+
 tasks.kubernetes.namespace = "default"
 
 # list groups of 5 values

diff --git a/core/src/main/scala/tasks/elastic/Deployment.scala b/core/src/main/scala/tasks/elastic/Deployment.scala
@@ -67,7 +67,7 @@ object Deployment {
         s"-Dhosts.gpusAsCommaString=${gpus.map(_.toString).mkString(",")}"
       else ""
 
-    val hostImageString = if (image.isDefined) s"-Dhosts.image=$image" else ""
+    val hostImageString = if (image.isDefined) s"-Dhosts.image=${image.get}" else ""
 
     val edited =
       s"./$packageFileName -J-Xmx{RAM}M -Dtasks.elastic.engine={GRID} {EXTRA} -Dhosts.master={MASTER} -Dhosts.app=false -Dtasks.fileservice.storageURI={STORAGE} -Dhosts.numCPU=$cpu -Dhosts.RAM=$memory -Dhosts.scratch=$scratch $gpuString $hostnameString $hostImageString"

diff --git a/core/src/main/scala/tasks/util/config/TasksConfig.scala b/core/src/main/scala/tasks/util/config/TasksConfig.scala
@@ -250,6 +250,7 @@ class TasksConfig(load: () => Config) {
   val appUIServerPort = raw.getInt("tasks.ui.app.port")
 
   def kubernetesImageName = raw.getString("tasks.kubernetes.image")
+  val kubernetesImageApplicationSubPath = raw.getString("tasks.kubernetes.imageApplicationSubPath")
 
   def kubernetesHostNameOrIPEnvVar =
     raw.getString("tasks.kubernetes.hostnameOrIPEnvVar")

diff --git a/kubernetes-test/src/main/scala/KubernetesTest.scala b/kubernetes-test/src/main/scala/KubernetesTest.scala
@@ -57,7 +57,6 @@ object KubernetesTest extends IOApp {
       IO(1)
   }
 
-  // tasks.worker-main-class = "tasks.KubernetesTestSlave"
   val testConfig2 = {
     val tmp = tasks.util.TempFile.createTempFile(".temp")
     tmp.delete
@@ -69,7 +68,7 @@ object KubernetesTest extends IOApp {
       tasks.elastic.queueCheckInterval = 3 seconds  
       tasks.addShutdownHook = false
       tasks.failuredetector.acceptable-heartbeat-pause = 10 s
-      tasks.kubernetes.image = "eclipse-temurin-unzip"
+      tasks.kubernetes.image = "eclipse-temurin:17.0.13_11-jre-ubi9-minimal"
       tasks.kubernetes.image-pull-policy = "IfNotPresent"
       
       tasks.kubernetes.podSpec = {

diff --git a/kubernetes/src/main/scala/tasks/elastic/kubernetes/Bootstrap.scala b/kubernetes/src/main/scala/tasks/elastic/kubernetes/Bootstrap.scala
@@ -117,9 +117,12 @@ object Bootstrap {
       scribe.info("No MY_POD_IP env found. Create pod of master.")
       k8sClientResource
         .flatMap { k8s =>
+          val pathOfEntrypointInBootstrapContainer =
+            tconfig.kubernetesImageApplicationSubPath
           val container = selfpackage.jib.containerize(
             out = addScribe(containerizer),
-            mainClassNameArg = Some(mainClassName)
+            mainClassNameArg = Some(mainClassName),
+            pathInContainer = pathOfEntrypointInBootstrapContainer
           )
 
           scribe.info(
@@ -133,7 +136,7 @@ object Bootstrap {
 
           val kubeCPURequest = userCPURequest + tconfig.kubernetesCpuExtra
           val kubeRamRequest = userRamRequest + tconfig.kubernetesRamExtra
-          val podName = ("main-" + KubernetesHelpers.newName).take(47)
+          val podName = ("tasks-app-" + KubernetesHelpers.newName).take(47)
 
           val imageName = container.getTargetImage().toString
 
@@ -145,6 +148,7 @@ object Bootstrap {
             }
             .getOrElse(PodSpec(containers = Nil))
 
+          val containerName = "tasks-app"
           val resource = Pod(
             metadata = Some(
               ObjectMeta(
@@ -160,8 +164,13 @@ object Bootstrap {
                 containers = List(
                   Container(
                     image = Some(imageName),
-                    command = Some(List("bash", "/app/entrypoint.sh")),
-                    name = "tasks-master",
+                    command = Some(
+                      List(
+                        "bash",
+                        s"$pathOfEntrypointInBootstrapContainer/entrypoint.sh"
+                      )
+                    ),
+                    name = containerName,
                     imagePullPolicy = Some(tconfig.kubernetesImagePullPolicy),
                     env = Some(
                       List(
@@ -237,23 +246,36 @@ object Bootstrap {
                 }
 
               val podIsRunning = phaseStream
-                .takeWhile(either => !either.exists(_.exists(_ == "Running")))
+                .takeThrough(either =>
+                  either match {
+                    case Left(error) =>
+                      scribe.error(error)
+                      false
+                    case Right(None) =>
+                      scribe.error("No phase in pod status")
+                      false
+                    case Right(Some(phase)) =>
+                      phase != "Running" && phase != "Failed"
+                  }
+                )
                 .evalTap(phase =>
                   IO { scribe.info(s"$podName in phase $phase") }
                 )
                 .compile
-                .drain
-                .flatTap(_ =>
-                  IO { scribe.info(s"$podName reached Running phase. ") }
+                .last
+                .map(option =>
+                  option.exists(_.exists(_.exists(_ == "Running")))
                 )
 
               fs2.Stream
                 .eval(podIsRunning)
-                .flatMap(_ =>
-                  k8s.pods
-                    .namespace(tconfig.kubernetesNamespace)
-                    .log(podName, Some("tasks-master"), follow = true)
-                    .flatMap(response => response.bodyText)
+                .flatMap(podIsRunning =>
+                  if (podIsRunning)
+                    k8s.pods
+                      .namespace(tconfig.kubernetesNamespace)
+                      .log(podName, Some(containerName), follow = true)
+                      .flatMap(response => response.bodyText)
+                  else fs2.Stream.empty
                 )
             } else {
               scribe.error("Failed pod creation")
@@ -280,7 +302,10 @@ object Bootstrap {
               .flatMap(_ =>
                 k8s.pods.namespace(tconfig.kubernetesNamespace).delete(podName)
               )
-              .map(_ => None)
+              .map(deletionStatus => {
+                scribe.info(s"Deletion status $deletionStatus")
+                None
+              })
           )
 
         }

diff --git a/project/metals.sbt b/project/metals.sbt
@@ -3,6 +3,6 @@
 
 // This file enables sbt-bloop to create bloop config files.
 
-addSbtPlugin("ch.epfl.scala" % "sbt-bloop" % "2.0.3")
+addSbtPlugin("ch.epfl.scala" % "sbt-bloop" % "2.0.8")
 
 // format: on