From 9fab60ba7b4c9b18f26fc455f74af3d12635a075 Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Thu, 10 Oct 2019 08:59:16 +0200 Subject: [PATCH 1/6] support newer snakeMake --- conda-recipe/meta.yaml | 2 +- docs/content/News.rst | 5 +++++ requirements.txt | 2 +- setup.py | 3 +++ snakePipes/workflows/ATAC-seq/Snakefile | 2 +- snakePipes/workflows/ChIP-seq/Snakefile | 2 +- snakePipes/workflows/DNA-mapping/Snakefile | 2 +- snakePipes/workflows/HiC/Snakefile | 2 +- snakePipes/workflows/RNA-seq/Snakefile | 2 +- snakePipes/workflows/WGBS/Snakefile | 2 +- snakePipes/workflows/createIndices/Snakefile | 2 +- snakePipes/workflows/preprocessing/Snakefile | 2 +- snakePipes/workflows/scRNAseq/Snakefile | 2 +- 13 files changed, 19 insertions(+), 11 deletions(-) diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index 6de3642c0..f04c2c72c 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -14,7 +14,7 @@ requirements: - python >=3 run: - python >=3 - - snakemake >=5.2.3 + - snakemake >=5.7 - pandas - graphviz - fuzzywuzzy diff --git a/docs/content/News.rst b/docs/content/News.rst index 1125df8ca..152f16022 100644 --- a/docs/content/News.rst +++ b/docs/content/News.rst @@ -1,6 +1,11 @@ snakePipes News =============== +snakePipes 1.3.1 +---------------- + + * Support for snakeMake 5.7.0 + snakePipes 1.3.0 ---------------- diff --git a/requirements.txt b/requirements.txt index a8a8a5c2e..ca4727c30 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -snakemake >= 5.2.3 +snakemake >= 5.7.0 psutil pandas fuzzywuzzy diff --git a/setup.py b/setup.py index 98d39bf0f..65ac2a4c5 100755 --- a/setup.py +++ b/setup.py @@ -10,12 +10,15 @@ for d in glob('snakePipes/workflows/*'): scripts.append(os.path.join(d, os.path.split(d)[1])) +requires = open("requirements.txt").read().strip().split("\n") + setup( name='snakePipes', version=__version__, # noqa: F821 scripts=scripts, packages=find_packages(), include_package_data=True, + install_requires=requires, url='https://github.com/maxplanck-ie/snakepipes', license='MIT', description='Snakemake workflows and wrappers for NGS data processing from the MPI-IE', diff --git a/snakePipes/workflows/ATAC-seq/Snakefile b/snakePipes/workflows/ATAC-seq/Snakefile index 0a29d64ba..b3125a474 100755 --- a/snakePipes/workflows/ATAC-seq/Snakefile +++ b/snakePipes/workflows/ATAC-seq/Snakefile @@ -11,7 +11,7 @@ workflow_rscripts=os.path.join(maindir, "shared", "rscripts") globals().update(cf.set_env_yamls()) # load config file -globals().update(cf.load_configfile(workflow.overwrite_configfile, config["verbose"])) +globals().update(cf.load_configfile(workflow.overwrite_configfiles, config["verbose"])) # load organism-specific data, i.e. genome indices, annotation, etc. globals().update(cf.load_organism_data(genome, maindir, config["verbose"])) diff --git a/snakePipes/workflows/ChIP-seq/Snakefile b/snakePipes/workflows/ChIP-seq/Snakefile index 15202fc33..b120d3565 100755 --- a/snakePipes/workflows/ChIP-seq/Snakefile +++ b/snakePipes/workflows/ChIP-seq/Snakefile @@ -11,7 +11,7 @@ workflow_rscripts=os.path.join(maindir, "shared", "rscripts") globals().update(cf.set_env_yamls()) # load config file -globals().update(cf.load_configfile(workflow.overwrite_configfile, config["verbose"])) +globals().update(cf.load_configfile(workflow.overwrite_configfiles, config["verbose"])) # load organism-specific data, i.e. genome indices, annotation, etc. globals().update(cf.load_organism_data(genome, maindir, config["verbose"])) diff --git a/snakePipes/workflows/DNA-mapping/Snakefile b/snakePipes/workflows/DNA-mapping/Snakefile index 4b2e0235d..5c533eaaa 100755 --- a/snakePipes/workflows/DNA-mapping/Snakefile +++ b/snakePipes/workflows/DNA-mapping/Snakefile @@ -8,7 +8,7 @@ maindir = os.path.dirname(os.path.dirname(workflow.basedir)) globals().update(cf.set_env_yamls()) # load config file -globals().update(cf.load_configfile(workflow.overwrite_configfile, config["verbose"])) +globals().update(cf.load_configfile(workflow.overwrite_configfiles, config["verbose"])) # load organism-specific data, i.e. genome indices, annotation, etc. globals().update(cf.load_organism_data(genome, maindir, config["verbose"])) diff --git a/snakePipes/workflows/HiC/Snakefile b/snakePipes/workflows/HiC/Snakefile index 85d5fb17c..447d73a05 100755 --- a/snakePipes/workflows/HiC/Snakefile +++ b/snakePipes/workflows/HiC/Snakefile @@ -9,7 +9,7 @@ maindir = os.path.dirname(os.path.dirname(workflow.basedir)) globals().update(cf.set_env_yamls()) # load config file -globals().update(cf.load_configfile(workflow.overwrite_configfile, config["verbose"])) +globals().update(cf.load_configfile(workflow.overwrite_configfiles, config["verbose"])) # load organism-specific data, i.e. genome indices, annotation, etc. globals().update(cf.load_organism_data(genome, maindir, config["verbose"])) diff --git a/snakePipes/workflows/RNA-seq/Snakefile b/snakePipes/workflows/RNA-seq/Snakefile index 84adf6b92..a66fd229a 100755 --- a/snakePipes/workflows/RNA-seq/Snakefile +++ b/snakePipes/workflows/RNA-seq/Snakefile @@ -9,7 +9,7 @@ maindir = os.path.dirname(os.path.dirname(workflow.basedir)) globals().update(cf.set_env_yamls()) # load config file -globals().update(cf.load_configfile(workflow.overwrite_configfile, config["verbose"])) +globals().update(cf.load_configfile(workflow.overwrite_configfiles, config["verbose"])) # load organism-specific data, i.e. genome indices, annotation, etc. globals().update(cf.load_organism_data(genome, maindir, config["verbose"])) diff --git a/snakePipes/workflows/WGBS/Snakefile b/snakePipes/workflows/WGBS/Snakefile index 43c0d60fa..cd45b965c 100644 --- a/snakePipes/workflows/WGBS/Snakefile +++ b/snakePipes/workflows/WGBS/Snakefile @@ -15,7 +15,7 @@ workflow_rscripts=os.path.join(maindir, "shared", "rscripts") globals().update(cf.set_env_yamls()) # load config file -globals().update(cf.load_configfile(workflow.overwrite_configfile, config["verbose"])) +globals().update(cf.load_configfile(workflow.overwrite_configfiles, config["verbose"])) # load organism-specific data, i.e. genome indices, annotation, etc. globals().update(cf.load_organism_data(genome, maindir, config["verbose"])) diff --git a/snakePipes/workflows/createIndices/Snakefile b/snakePipes/workflows/createIndices/Snakefile index 20497343a..f633bd4e7 100644 --- a/snakePipes/workflows/createIndices/Snakefile +++ b/snakePipes/workflows/createIndices/Snakefile @@ -8,7 +8,7 @@ maindir = os.path.dirname(os.path.dirname(workflow.basedir)) globals().update(cf.set_env_yamls()) # load config file -globals().update(cf.load_configfile(workflow.overwrite_configfile, config["verbose"])) +globals().update(cf.load_configfile(workflow.overwrite_configfiles, config["verbose"])) ### include modules of other snakefiles ######################################## ################################################################################ diff --git a/snakePipes/workflows/preprocessing/Snakefile b/snakePipes/workflows/preprocessing/Snakefile index 35e24444a..5bb6196dd 100755 --- a/snakePipes/workflows/preprocessing/Snakefile +++ b/snakePipes/workflows/preprocessing/Snakefile @@ -9,7 +9,7 @@ maindir = os.path.dirname(os.path.dirname(workflow.basedir)) globals().update(cf.set_env_yamls()) # load config file -globals().update(cf.load_configfile(workflow.overwrite_configfile, config["verbose"])) +globals().update(cf.load_configfile(workflow.overwrite_configfiles, config["verbose"])) # do workflow specific stuff now include: os.path.join(workflow.basedir, "internals.snakefile") diff --git a/snakePipes/workflows/scRNAseq/Snakefile b/snakePipes/workflows/scRNAseq/Snakefile index d7fabc316..e0e44c21f 100755 --- a/snakePipes/workflows/scRNAseq/Snakefile +++ b/snakePipes/workflows/scRNAseq/Snakefile @@ -9,7 +9,7 @@ workflow_rscripts=os.path.join(maindir, "shared", "rscripts") globals().update(cf.set_env_yamls()) # load config file -globals().update(cf.load_configfile(workflow.overwrite_configfile, config["verbose"])) +globals().update(cf.load_configfile(workflow.overwrite_configfiles, config["verbose"])) # load organism-specific data, i.e. genome indices, annotation, etc. globals().update(cf.load_organism_data(genome, maindir, config["verbose"])) From 124318d773cfaec61f85f1dcea3d980e1de9ba60 Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Thu, 10 Oct 2019 08:59:39 +0200 Subject: [PATCH 2/6] bump version --- snakePipes/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snakePipes/__init__.py b/snakePipes/__init__.py index 19b4f1d60..72837bdc7 100644 --- a/snakePipes/__init__.py +++ b/snakePipes/__init__.py @@ -1 +1 @@ -__version__ = '1.3.0' +__version__ = '1.3.1' From faac1f9e193a776ab9b430c270cb96097073e89d Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Thu, 10 Oct 2019 09:00:20 +0200 Subject: [PATCH 3/6] missed one --- conda-recipe/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index f04c2c72c..b20168217 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: snakepipes - version: 1.3.0 + version: 1.3.1 source: path: ../ From 5dd94c619ba79c4a17b47910010d34a98bf8d4d8 Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Thu, 10 Oct 2019 09:09:50 +0200 Subject: [PATCH 4/6] debug --- snakePipes/common_functions.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/snakePipes/common_functions.py b/snakePipes/common_functions.py index 85a8668b4..64f8064ad 100644 --- a/snakePipes/common_functions.py +++ b/snakePipes/common_functions.py @@ -49,15 +49,16 @@ def sanity_dict_clean(myDict): return myDict -def load_configfile(configFile, verbose, info='Config'): - with open(configFile, "r") as f: +def load_configfile(configFiles, verbose, info='Config'): + print("configFiles is {}".format(configFiles)) + with open(configFiles, "r") as f: config = yaml.load(f, Loader=yaml.FullLoader) config = sanity_dict_clean(config) if verbose: print("\n--- " + info + " ---------------------------------------------------------------------") - print("config file: {}".format(configFile)) + print("config file: {}".format(configFiles)) for k, v in sorted(config.items()): print("{}: {}".format(k, v)) print("-" * 80, "\n") From 8805194565b199e763e90f64c08b102cee69542d Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Thu, 10 Oct 2019 09:19:16 +0200 Subject: [PATCH 5/6] ensure just one item --- snakePipes/common_functions.py | 1 - snakePipes/workflows/ATAC-seq/Snakefile | 2 +- snakePipes/workflows/ChIP-seq/Snakefile | 2 +- snakePipes/workflows/DNA-mapping/Snakefile | 2 +- snakePipes/workflows/HiC/Snakefile | 2 +- snakePipes/workflows/RNA-seq/Snakefile | 2 +- snakePipes/workflows/WGBS/Snakefile | 2 +- snakePipes/workflows/createIndices/Snakefile | 2 +- snakePipes/workflows/preprocessing/Snakefile | 2 +- snakePipes/workflows/scRNAseq/Snakefile | 2 +- 10 files changed, 9 insertions(+), 10 deletions(-) diff --git a/snakePipes/common_functions.py b/snakePipes/common_functions.py index 64f8064ad..a7b92ac42 100644 --- a/snakePipes/common_functions.py +++ b/snakePipes/common_functions.py @@ -50,7 +50,6 @@ def sanity_dict_clean(myDict): def load_configfile(configFiles, verbose, info='Config'): - print("configFiles is {}".format(configFiles)) with open(configFiles, "r") as f: config = yaml.load(f, Loader=yaml.FullLoader) diff --git a/snakePipes/workflows/ATAC-seq/Snakefile b/snakePipes/workflows/ATAC-seq/Snakefile index b3125a474..c594c9344 100755 --- a/snakePipes/workflows/ATAC-seq/Snakefile +++ b/snakePipes/workflows/ATAC-seq/Snakefile @@ -11,7 +11,7 @@ workflow_rscripts=os.path.join(maindir, "shared", "rscripts") globals().update(cf.set_env_yamls()) # load config file -globals().update(cf.load_configfile(workflow.overwrite_configfiles, config["verbose"])) +globals().update(cf.load_configfile(workflow.overwrite_configfiles[0], config["verbose"])) # load organism-specific data, i.e. genome indices, annotation, etc. globals().update(cf.load_organism_data(genome, maindir, config["verbose"])) diff --git a/snakePipes/workflows/ChIP-seq/Snakefile b/snakePipes/workflows/ChIP-seq/Snakefile index b120d3565..dc3ec1a4b 100755 --- a/snakePipes/workflows/ChIP-seq/Snakefile +++ b/snakePipes/workflows/ChIP-seq/Snakefile @@ -11,7 +11,7 @@ workflow_rscripts=os.path.join(maindir, "shared", "rscripts") globals().update(cf.set_env_yamls()) # load config file -globals().update(cf.load_configfile(workflow.overwrite_configfiles, config["verbose"])) +globals().update(cf.load_configfile(workflow.overwrite_configfiles[0], config["verbose"])) # load organism-specific data, i.e. genome indices, annotation, etc. globals().update(cf.load_organism_data(genome, maindir, config["verbose"])) diff --git a/snakePipes/workflows/DNA-mapping/Snakefile b/snakePipes/workflows/DNA-mapping/Snakefile index 5c533eaaa..aec47e604 100755 --- a/snakePipes/workflows/DNA-mapping/Snakefile +++ b/snakePipes/workflows/DNA-mapping/Snakefile @@ -8,7 +8,7 @@ maindir = os.path.dirname(os.path.dirname(workflow.basedir)) globals().update(cf.set_env_yamls()) # load config file -globals().update(cf.load_configfile(workflow.overwrite_configfiles, config["verbose"])) +globals().update(cf.load_configfile(workflow.overwrite_configfiles[0], config["verbose"])) # load organism-specific data, i.e. genome indices, annotation, etc. globals().update(cf.load_organism_data(genome, maindir, config["verbose"])) diff --git a/snakePipes/workflows/HiC/Snakefile b/snakePipes/workflows/HiC/Snakefile index 447d73a05..6f0b1d5f5 100755 --- a/snakePipes/workflows/HiC/Snakefile +++ b/snakePipes/workflows/HiC/Snakefile @@ -9,7 +9,7 @@ maindir = os.path.dirname(os.path.dirname(workflow.basedir)) globals().update(cf.set_env_yamls()) # load config file -globals().update(cf.load_configfile(workflow.overwrite_configfiles, config["verbose"])) +globals().update(cf.load_configfile(workflow.overwrite_configfiles[0], config["verbose"])) # load organism-specific data, i.e. genome indices, annotation, etc. globals().update(cf.load_organism_data(genome, maindir, config["verbose"])) diff --git a/snakePipes/workflows/RNA-seq/Snakefile b/snakePipes/workflows/RNA-seq/Snakefile index a66fd229a..44fc6bb8e 100755 --- a/snakePipes/workflows/RNA-seq/Snakefile +++ b/snakePipes/workflows/RNA-seq/Snakefile @@ -9,7 +9,7 @@ maindir = os.path.dirname(os.path.dirname(workflow.basedir)) globals().update(cf.set_env_yamls()) # load config file -globals().update(cf.load_configfile(workflow.overwrite_configfiles, config["verbose"])) +globals().update(cf.load_configfile(workflow.overwrite_configfiles[0], config["verbose"])) # load organism-specific data, i.e. genome indices, annotation, etc. globals().update(cf.load_organism_data(genome, maindir, config["verbose"])) diff --git a/snakePipes/workflows/WGBS/Snakefile b/snakePipes/workflows/WGBS/Snakefile index cd45b965c..d5dd22899 100644 --- a/snakePipes/workflows/WGBS/Snakefile +++ b/snakePipes/workflows/WGBS/Snakefile @@ -15,7 +15,7 @@ workflow_rscripts=os.path.join(maindir, "shared", "rscripts") globals().update(cf.set_env_yamls()) # load config file -globals().update(cf.load_configfile(workflow.overwrite_configfiles, config["verbose"])) +globals().update(cf.load_configfile(workflow.overwrite_configfiles[0], config["verbose"])) # load organism-specific data, i.e. genome indices, annotation, etc. globals().update(cf.load_organism_data(genome, maindir, config["verbose"])) diff --git a/snakePipes/workflows/createIndices/Snakefile b/snakePipes/workflows/createIndices/Snakefile index f633bd4e7..ae2e9cf5f 100644 --- a/snakePipes/workflows/createIndices/Snakefile +++ b/snakePipes/workflows/createIndices/Snakefile @@ -8,7 +8,7 @@ maindir = os.path.dirname(os.path.dirname(workflow.basedir)) globals().update(cf.set_env_yamls()) # load config file -globals().update(cf.load_configfile(workflow.overwrite_configfiles, config["verbose"])) +globals().update(cf.load_configfile(workflow.overwrite_configfiles[0], config["verbose"])) ### include modules of other snakefiles ######################################## ################################################################################ diff --git a/snakePipes/workflows/preprocessing/Snakefile b/snakePipes/workflows/preprocessing/Snakefile index 5bb6196dd..713cc2b54 100755 --- a/snakePipes/workflows/preprocessing/Snakefile +++ b/snakePipes/workflows/preprocessing/Snakefile @@ -9,7 +9,7 @@ maindir = os.path.dirname(os.path.dirname(workflow.basedir)) globals().update(cf.set_env_yamls()) # load config file -globals().update(cf.load_configfile(workflow.overwrite_configfiles, config["verbose"])) +globals().update(cf.load_configfile(workflow.overwrite_configfiles[0], config["verbose"])) # do workflow specific stuff now include: os.path.join(workflow.basedir, "internals.snakefile") diff --git a/snakePipes/workflows/scRNAseq/Snakefile b/snakePipes/workflows/scRNAseq/Snakefile index e0e44c21f..f14b57b21 100755 --- a/snakePipes/workflows/scRNAseq/Snakefile +++ b/snakePipes/workflows/scRNAseq/Snakefile @@ -9,7 +9,7 @@ workflow_rscripts=os.path.join(maindir, "shared", "rscripts") globals().update(cf.set_env_yamls()) # load config file -globals().update(cf.load_configfile(workflow.overwrite_configfiles, config["verbose"])) +globals().update(cf.load_configfile(workflow.overwrite_configfiles[0], config["verbose"])) # load organism-specific data, i.e. genome indices, annotation, etc. globals().update(cf.load_organism_data(genome, maindir, config["verbose"])) From b9bafecadb612e5bd9aa89cbb17fb8396d1cb7fa Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Thu, 10 Oct 2019 09:24:04 +0200 Subject: [PATCH 6/6] debug azure on OSX --- azure-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 66b09c17f..09e7420d2 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -89,6 +89,7 @@ jobs: displayName: Add conda to PATH - bash: | if [ -f /Users/vsts/.conda/pkgs/urls.txt ]; then + echo "chown" sudo chown 501:20 /Users/vsts/.conda/pkgs/urls.txt fi displayName: Fix OSX permissions