[feat] Assemble can use conda-lock to create the environment (#35)

* [feat] Assemble can use conda-lock to create the environment

Miscellaneous:
* [test] add more unit tests
* [ci] add conda to github actions
* [ci] add conda caching
* [feat] move Process (previously used only by Solve) to it's own class
* [feat] refactor BuildWriter to support building conda environments using both `conda env create` and `conda-lock`
* [test] add a unit test for Solve; I could not get this to work on github actions, so it works locally only

.github/workflows/build.yml

@@ -16,10 +16,21 @@ jobs:
       matrix:
         java-version: [8, 11]
     runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -el {0}
     steps:
       - uses: actions/checkout@v2
         with:
           fetch-depth: 0
+      - name: Install Conda environment with Micromamba
+        uses: mamba-org/provision-with-micromamba@main
+        with:
+          cache-downloads: true
+          channels: "conda-forge,defaults"
+          environment-file: false
+          environment-name: conda-env-builder-test
+          extra-specs: "conda-lock=1.4.0"
       - uses: actions/setup-java@v1
         with:
           java-version: ${{ matrix.java-version }}
@@ -31,7 +42,7 @@ jobs:
           git config --add user.email "mill-ci@localhost"
           ./mill --jobs 2 clean
           ./mill --jobs 2 --disable-ticker -s _.compile
-          ./mill --jobs 2 --disable-ticker -s tools.test
+          ./mill --jobs 2 --disable-ticker -s tools.test.testOnly -- -l ExcludeGithubActions
           ./mill --jobs 2 --disable-ticker -s tools.scoverage.xmlReport
           ./mill --jobs 2 --disable-ticker -s tools.scoverage.htmlReport
           bash <(curl -s https://codecov.io/bash) -c -F tools -f '!*.txt'

README.md

@@ -50,14 +50,12 @@ Build and maintain multiple custom conda environments all in once place.
 Install with [`conda`](https://conda.io/projects/conda/en/latest/index.html): `conda install --channel conda-forge conda-env-builder`.
 
 
-
 ## Goals
 
-
-* Specify multiple environments in one place
+* Specify multiple conda environments in one place
 * Reduce duplication with cross-environment defaults and environment inheritance
-* Install `pip` packages into your conda environment, as well as custom commands
-* Produce easy scripts to build your environments
+* Produce easy scripts to build your environments (using `conda env create` or `conda-lock install`)
+* Install `pip` packages into your conda environment, as well as custom commands (e.g. `git clone ... && make install`)
 
 ## Overview
 
@@ -90,7 +88,8 @@ Below we highlight a few tools that you may find useful.
 * `Assemble`: builds per-environment conda environment and custom command build scripts.
   * Builds `<env-name>.yaml` for your conda+pip environment specification YAML.
   * Builds `<env-name>.build-conda.sh` to build your conda environment.
-  * Builds `<env-name>.build-local.sh` to execute any custom commands after creating the conda envirnment.
+  * Builds `<env-name>.build-local.sh` to execute any custom commands after creating the conda environment.
+  * Builds [conda-lock](https://github.com/conda/conda-lock) environment YAML to `<output>/<env-name>.<platform.conda-lock.yml`, if `--conda-lock=<platform>` is specified
 * `Solve`: updates the configuration with a full list of packages and versions for the environment.
   * For each environment, builds it (`conda env create`), exports it (`conda env export`), and update the specification
 * `Tabulate`: writes the specification in a tabular format. 

tools/src/com/github/condaincubator/condaenvbuilder/CondaEnvironmentBuilderDef.scala

@@ -1,9 +1,9 @@
 package com.github.condaincubator.condaenvbuilder
 
-import java.nio.file.Path
-
 import com.fulcrumgenomics.commons.CommonsDef
 
+import java.nio.file.Path
+
 /**
   * Object that is designed to be imported with `import CondaEnvironmentBuilderDef._` in any/all classes
   * much like the way that scala.PreDef is imported in all files automatically.

tools/src/com/github/condaincubator/condaenvbuilder/cmdline/CondaEnvironmentBuilderMain.scala

@@ -39,7 +39,7 @@ class CondaEnvironmentBuilderCommonArgs
 
   Logger.level = this.logLevel
   CondaEnvironmentBuilderTool.UseMamba = mamba
-  CondaEnvironmentBuilderTool.FileExtension = extension
+  CondaEnvironmentBuilderTool.YamlFileExtension = extension
 }
 
 class CondaEnvironmentBuilderMain extends LazyLogging {

tools/src/com/github/condaincubator/condaenvbuilder/cmdline/CondaEnvironmentBuilderTool.scala

@@ -2,14 +2,18 @@ package com.github.condaincubator.condaenvbuilder.cmdline
 
 import com.fulcrumgenomics.commons.util.LazyLogging
 import com.fulcrumgenomics.sopt.cmdline.ValidationException
-import CondaEnvironmentBuilderMain.FailureException
+import com.github.condaincubator.condaenvbuilder.cmdline.CondaEnvironmentBuilderMain.FailureException
 
 object CondaEnvironmentBuilderTool {
   /** True to use `mamba` instead of `conda`, false otherwise. */
   var UseMamba: Boolean = false
 
+  /** True to use `micromamba` instead of `conda` or `mamba`, false otherwise. Needed for testing in micromamba
+   * environments*/
+  var UseMicromamba: Boolean = false
+
   /** The file extension to use for YAML files. */
-  var FileExtension: String = "yml"
+  var YamlFileExtension: String = "yml"
 }
 
 
@@ -33,6 +37,10 @@ trait CondaEnvironmentBuilderTool extends LazyLogging {
   def validate(test: Boolean, message: => String): Unit = if (!test) throw new ValidationException(message)
 
   /** Returns the conda executable to use. */
-  protected def condaExecutable: String = if (CondaEnvironmentBuilderTool.UseMamba) "mamba" else "conda"
+  protected def condaExecutable: String = {
+    if (CondaEnvironmentBuilderTool.UseMicromamba) "micromamba"
+    else if (CondaEnvironmentBuilderTool.UseMamba) "mamba"
+    else "conda"
+  }
 }
 

tools/src/com/github/condaincubator/condaenvbuilder/io/BuildWriter.scala

@@ -5,76 +5,52 @@ import com.fulcrumgenomics.commons.io.Io
 import com.fulcrumgenomics.commons.util.{LazyLogging, Logger}
 import com.github.condaincubator.condaenvbuilder.CondaEnvironmentBuilderDef.PathToYaml
 import com.github.condaincubator.condaenvbuilder.api.CodeStep.Command
+import com.github.condaincubator.condaenvbuilder.api.CondaStep.Platform
 import com.github.condaincubator.condaenvbuilder.api.{CodeStep, CondaStep, Environment, PipStep}
 import com.github.condaincubator.condaenvbuilder.cmdline.CondaEnvironmentBuilderTool
 
 import java.io.PrintWriter
 import java.nio.file.Paths
 
-/** Companion to [[BuildWriter]].  */
-object BuildWriter {
+trait BuildWriterConstants {
 
   /** Returns the path to the environment's conda YAML. */
-  private def toEnvironmentYaml(environment: Environment, output: DirPath): PathToYaml = {
-    output.resolve(f"${environment.name}.${CondaEnvironmentBuilderTool.FileExtension}")
+  protected def toEnvironmentYaml(environment: Environment, output: DirPath): PathToYaml = {
+    output.resolve(f"${environment.name}.${CondaEnvironmentBuilderTool.YamlFileExtension}")
   }
 
+  /** Returns the path to the environment's conda LOCK file. */
+  protected def toEnvironmentLockYaml(environment: Environment, platform: Platform, output: DirPath): PathToYaml = {
+    output.resolve(f"${environment.name}.${platform}.conda-lock.${CondaEnvironmentBuilderTool.YamlFileExtension}")
+  }
+
+
   /** Returns the path to the environment's conda build script. */
-  private def toCondaBuildScript(environment: Environment, output: DirPath): FilePath = {
+  protected def toCondaBuildScript(environment: Environment, output: DirPath): FilePath = {
     output.resolve(f"${environment.name}.build-conda.sh")
   }
 
   /** Returns the path to the environment's custom code build script. */
-  private def toCodeBuildScript(environment: Environment, output: DirPath): FilePath = {
+  protected def toCodeBuildScript(environment: Environment, output: DirPath): FilePath = {
     output.resolve(f"${environment.name}.build-local.sh")
   }
-
-  /** Builds a new [[BuildWriter]] for the given environment.
-    *
-    * @param environment the environment for which build files should be created.
-    * @param output the output directory where build files should be created.
-    * @param environmentYaml the path to use for the environment's conda YAML, otherwise `<output>/<env-name>.yml`.
-    * @param condaBuildScript the path to use for the environment's conda build script,
-    *                         otherwise `<output>/<env-name>.build-conda.sh`.
-    * @param codeBuildScript the path to use for the environment's custom code build script,
-    *                        otherwise `<output>/<env-name>.build-local.sh`.
-    * @param condaEnvironmentDirectory the directory in which conda environments should be stored when created.
-    * @return
-    */
-  def apply(environment: Environment,
-            output: DirPath,
-            environmentYaml: Option[PathToYaml] = None,
-            condaBuildScript: Option[FilePath] = None,
-            codeBuildScript: Option[FilePath] = None,
-            condaEnvironmentDirectory: Option[DirPath] = None): BuildWriter = {
-    BuildWriter(
-      environment               = environment,
-      environmentYaml           = environmentYaml.getOrElse(toEnvironmentYaml(environment, output)),
-      condaBuildScript          = condaBuildScript.getOrElse(toCondaBuildScript(environment, output)),
-      codeBuildScript           = codeBuildScript.getOrElse(toCodeBuildScript(environment, output)),
-      condaEnvironmentDirectory = condaEnvironmentDirectory
-    )
-  }
 }
 
-/** Writer that is used to create the build scripts for the conda environments.
-  *
-  * The conda build script should be executed first, then the custom code build script.  The conda environment
-  * specification is stored in the given environment YAML path.
-  *
-  * @param environment the environment for which build files should be created.
-  * @param environmentYaml the path to use for the environment's conda YAML.
-  * @param condaBuildScript the path to use for the environment's conda build script
-  * @param codeBuildScript the path to use for the environment's custom code build script
-  * @param condaEnvironmentDirectory the directory in which conda environments should be stored when created.
-  */
-case class BuildWriter(environment: Environment,
-                       environmentYaml: PathToYaml,
-                       condaBuildScript: FilePath,
-                       codeBuildScript: FilePath,
-                       condaEnvironmentDirectory: Option[DirPath]) extends LazyLogging {
-
-  private lazy val condaExecutable: String = if (CondaEnvironmentBuilderTool.UseMamba) "mamba" else "conda"
+
+trait BuildWriter extends LazyLogging {
+  def environment: Environment
+
+  /** the path to use for the environment's conda YAML */
+  def environmentYaml: PathToYaml
+
+  /** The path to use for the environment's conda build script */
+  def condaBuildScript: FilePath
+
+  /** The path to use for the environment's custom code build script */
+  def codeBuildScript: FilePath
+
+  /** The directory in which conda environments should be stored when created */
+  def condaEnvironmentDirectory: Option[DirPath]
 
   /** Returns all the output files that will be written by this writer */
   def allOutputs: Iterable[FilePath] = Seq(environmentYaml, condaBuildScript, codeBuildScript)
@@ -91,20 +67,20 @@ case class BuildWriter(environment: Environment,
 
   /** Writes the conda environment file. */
   def writeEnvironmentYaml(logger: Logger = this.logger): Unit = {
-    logger.info(s"Writing the environment YAML for ${environment.name} to: $environmentYaml")
+    logger.info(s"Writing the conda environment YAML for ${environment.name} to: $environmentYaml")
 
     val condaStep: Option[CondaStep] = environment.steps.collect { case step: CondaStep => step } match {
-      case Seq()     => None
+      case Seq() => None
       case Seq(step) => Some(step)
-      case steps     => throw new IllegalArgumentException(
+      case steps => throw new IllegalArgumentException(
         s"Expected a single conda step, found ${steps.length} conda steps.  Did you forget to compile?"
       )
     }
 
     val pipStep: Option[PipStep] = environment.steps.collect { case step: PipStep => step } match {
-      case Seq()     => None
+      case Seq() => None
       case Seq(step) => Some(step)
-      case steps     => throw new IllegalArgumentException(
+      case steps => throw new IllegalArgumentException(
         s"Expected a single pip step, found ${steps.length} pip steps.  Did you forget to compile?"
       )
     }
@@ -139,8 +115,11 @@ case class BuildWriter(environment: Environment,
     writer.close()
   }
 
+  /** Write the conda build command. */
+  protected def writeCondaBuildCommand(writer: PrintWriter): Unit
+
   /** Writes the conda build script. */
-  def writeCondaBuildScript(logger: Logger = this.logger): Unit = {
+  private def writeCondaBuildScript(logger: Logger = this.logger): Unit = {
     logger.info(s"Writing conda build script for ${environment.name} to: $condaBuildScript")
     val writer = new PrintWriter(Io.toWriter(condaBuildScript))
     writer.println("#/bin/bash\n")
@@ -149,24 +128,19 @@ case class BuildWriter(environment: Environment,
     writer.println("# Move to the scripts directory")
     writer.println("pushd $(dirname $0)\n")
     writer.println("# Build the conda environment")
-    writer.write(f"$condaExecutable env create --force --verbose --quiet")
-    condaEnvironmentDirectory match {
-      case Some(pre) => writer.write(f" --prefix ${pre.toAbsolutePath}/${environment.name}")
-      case None      => writer.write(f" --name ${environment.name}")
-    }
-    writer.println(f" --file ${environmentYaml.toFile.getName}\n")
+    this.writeCondaBuildCommand(writer=writer)
     writer.println("popd\n")
     writer.close()
   }
 
   /** Writes the custom code build script. */
-  def writeCodeBuildScript(logger: Logger = this.logger): Unit = {
+  protected def writeCodeBuildScript(logger: Logger = this.logger): Unit = {
     logger.info(s"Writing custom code build script for ${environment.name} to: $codeBuildScript")
 
     val codeStep: Option[CodeStep] = environment.steps.collect { case step: CodeStep => step } match {
-      case Seq()     => None
+      case Seq() => None
       case Seq(step) => Some(step)
-      case steps     => throw new IllegalArgumentException(
+      case steps => throw new IllegalArgumentException(
         s"Expected a single code step, found ${steps.length} code steps.  Did you forget to compile?"
       )
     }
@@ -178,11 +152,11 @@ case class BuildWriter(environment: Environment,
     val buildPath = codeStep.map(_.path).getOrElse(Paths.get("."))
     writer.println(f"""repo_root=$${1:-"$buildPath"}\n""")
     codeStep match {
-      case None       => writer.println("# No custom commands")
+      case None => writer.println("# No custom commands")
       case Some(step) =>
         writer.println(f"# Activate conda environment: ${environment.name}")
-        writer.println("set +eu")  // because of unbound variables
-        writer.println("PS1=dummy\n")  // for sourcing
+        writer.println("set +eu") // because of unbound variables
+        writer.println("PS1=dummy\n") // for sourcing
         writer.println(f". $$(conda info --base | tail -n 1)/etc/profile.d/conda.sh") // tail to ignore mamba header
         writer.println(f"conda activate ${environment.name}")
         writer.println()

tools/src/com/github/condaincubator/condaenvbuilder/io/CondaBuildWriter.scala

@@ -0,0 +1,73 @@
+package com.github.condaincubator.condaenvbuilder.io
+
+import com.fulcrumgenomics.commons.CommonsDef.{DirPath, FilePath}
+import com.fulcrumgenomics.commons.io.Io
+import com.fulcrumgenomics.commons.util.Logger
+import com.github.condaincubator.condaenvbuilder.CondaEnvironmentBuilderDef.PathToYaml
+import com.github.condaincubator.condaenvbuilder.api.Environment
+import com.github.condaincubator.condaenvbuilder.cmdline.CondaEnvironmentBuilderTool
+
+import java.io.PrintWriter
+
+
+/** Companion to [[CondaBuildWriter]].  */
+object CondaBuildWriter extends BuildWriterConstants {
+
+  /** Builds a new [[CondaBuildWriter]] for the given environment.
+   *
+   * @param environment the environment for which build files should be created.
+   * @param output the output directory where build files should be created.
+   * @param environmentYaml the path to use for the environment's conda YAML, otherwise `<output>/<env-name>.yml`.
+   * @param condaBuildScript the path to use for the environment's conda build script,
+   *                         otherwise `<output>/<env-name>.build-conda.sh`.
+   * @param codeBuildScript the path to use for the environment's custom code build script,
+   *                        otherwise `<output>/<env-name>.build-local.sh`.
+   * @param condaEnvironmentDirectory the directory in which conda environments should be stored when created.
+   * @return
+   */
+  def apply(environment: Environment,
+            output: DirPath,
+            environmentYaml: Option[PathToYaml] = None,
+            condaBuildScript: Option[FilePath] = None,
+            codeBuildScript: Option[FilePath] = None,
+            condaEnvironmentDirectory: Option[DirPath] = None): CondaBuildWriter = {
+    CondaBuildWriter(
+      environment               = environment,
+      environmentYaml           = environmentYaml.getOrElse(toEnvironmentYaml(environment, output)),
+      condaBuildScript          = condaBuildScript.getOrElse(toCondaBuildScript(environment, output)),
+      codeBuildScript           = codeBuildScript.getOrElse(toCodeBuildScript(environment, output)),
+      condaEnvironmentDirectory = condaEnvironmentDirectory
+    )
+  }
+}
+
+
+/** Writer that is used to create the build scripts for the conda environments.
+ *
+ * The conda build script should be executed first, then the custom code build script.  The conda environment
+ * specification is stored in the given environment YAML path.
+ *
+ * @param environment the environment for which build files should be created.
+ * @param environmentYaml the path to use for the environment's conda YAML.
+ * @param condaBuildScript the path to use for the environment's conda build script
+ * @param codeBuildScript the path to use for the environment's custom code build script
+ * @param condaEnvironmentDirectory the directory in which conda environments should be stored when created.
+ */
+case class CondaBuildWriter(environment: Environment,
+                            environmentYaml: PathToYaml,
+                            condaBuildScript: FilePath,
+                            codeBuildScript: FilePath,
+                            condaEnvironmentDirectory: Option[DirPath]) extends BuildWriter {
+
+  private lazy val condaExecutable: String = if (CondaEnvironmentBuilderTool.UseMamba) "mamba" else "conda"
+
+  /** Writes the conda build command. */
+  protected def writeCondaBuildCommand(writer: PrintWriter): Unit = {
+    writer.write(f"$condaExecutable env create --force --verbose --quiet")
+    condaEnvironmentDirectory match {
+      case Some(pre) => writer.write(f" --prefix ${pre.toAbsolutePath}/${environment.name}")
+      case None      => writer.write(f" --name ${environment.name}")
+    }
+    writer.println(f" --file ${environmentYaml.toFile.getName}\n")
+  }
+}