Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use new, slimmer sometimes, containers #762

Merged
merged 10 commits into from
May 23, 2024
Merged

Use new, slimmer sometimes, containers #762

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
787f7d0
update to new scpcatools image
jashapiro May 8, 2024
ed48f8c
Add updated container versions
jashapiro May 8, 2024
e9f3346
Merge branch 'development' into jashapiro/nextgen-containers
jashapiro May 9, 2024
dc61e6f
Use slim containers & other specialized
jashapiro May 21, 2024
2fcefbc
update schema
jashapiro May 21, 2024
74527c8
revert make_unfiltered_sce to full model
jashapiro May 23, 2024
65cab57
Revert "revert make_unfiltered_sce to full model"
jashapiro May 23, 2024
52d152b
pre-commit updates
jashapiro May 23, 2024
1b3b100
Update modules/classify-celltypes.nf
jashapiro May 23, 2024
6512598
Use Seurat for cellhash demux
jashapiro May 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions .pre-commit-config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,28 +32,28 @@ repos:
args: [--update-only, --title=**Table of Contents**]
- repo: https://github.com/astral-sh/ruff-pre-commit
# Ruff for linting and formatting python
rev: v0.3.3
rev: v0.4.5
hooks:
# Run the linter.
- id: ruff
# Run the formatter.
- id: ruff-format
- repo: https://github.com/lorenzwalthert/precommit
# R styling and linting
rev: v0.4.0
rev: v0.4.2
hooks:
- id: style-files
args: [--style_pkg=styler, --style_fun=tidyverse_style]
# - id: lintr #skip R linting for now...
- id: parsable-R
- repo: https://github.com/pre-commit/mirrors-prettier
# Format YAML and other languages
rev: v3.0.3
rev: v3.1.0
hooks:
- id: prettier
exclude: '\.md$'
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.5.0
rev: v4.6.0
hooks:
- id: check-added-large-files
args: ["--maxkb=200"]
Expand Down
9 changes: 7 additions & 2 deletions config/containers.config
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
// Docker container images
SCPCATOOLS_CONTAINER = 'ghcr.io/alexslemonade/scpca-tools:v0.3.2'
SCPCATOOLS_CONTAINER = 'ghcr.io/alexslemonade/scpcatools:edge'
SCPCATOOLS_SLIM_CONTAINER = 'ghcr.io/alexslemonade/scpcatools-slim:edge'
SCPCATOOLS_ANNDATA_CONTAINER = 'ghcr.io/alexslemonade/scpcatools-anndata:edge'
SCPCATOOLS_REPORTS_CONTAINER = 'ghcr.io/alexslemonade/scpcatools-reports:edge'
SCPCATOOLS_SEURAT_CONTAINER = 'ghcr.io/alexslemonade/scpcatools-seurat:edge'
SCPCATOOLS_SCVI_CONTAINER = 'ghcr.io/alexslemonade/scpcatools-scvi:edge'

ALEVINFRY_CONTAINER = 'quay.io/biocontainers/alevin-fry:0.7.0--h9f5acd7_1'
BCFTOOLS_CONTAINER = 'quay.io/biocontainers/bcftools:1.14--h88f3f91_0'
Expand All @@ -8,7 +13,7 @@ FASTP_CONTAINER = 'quay.io/biocontainers/fastp:0.23.0--h79da9fb_0'
SALMON_CONTAINER = 'quay.io/biocontainers/salmon:1.9.0--h7e5ed60_1'
SAMTOOLS_CONTAINER = 'quay.io/biocontainers/samtools:1.14--hb421002_0'
STAR_CONTAINER = 'quay.io/biocontainers/star:2.7.9a--h9ee0642_0'
TIDYVERSE_CONTAINER = 'rocker/tidyverse:4.3.1'
TIDYVERSE_CONTAINER = 'rocker/tidyverse:4.4.0'
VIREO_CONTAINER = 'ghcr.io/alexslemonade/vireo-snp:v0.5.7'

// 10X software containers not set by default
Expand Down
6 changes: 3 additions & 3 deletions merge.nf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ if (param_error) {

// merge individual SCE objects into one SCE object
process merge_sce {
container params.SCPCATOOLS_CONTAINER
container params.SCPCATOOLS_SLIM_CONTAINER
tag "${merge_group_id}"
label 'mem_max'
label 'long_running'
Expand Down Expand Up @@ -60,7 +60,7 @@ process merge_sce {

// create merge report
process generate_merge_report {
container params.SCPCATOOLS_CONTAINER
container params.SCPCATOOLS_REPORTS_CONTAINER
tag "${merge_group_id}"
publishDir "${params.results_dir}/${merge_group_id}/merged"
label 'mem_max'
Expand Down Expand Up @@ -88,7 +88,7 @@ process generate_merge_report {
}

process export_anndata {
container params.SCPCATOOLS_CONTAINER
container params.SCPCATOOLS_ANNDATA_CONTAINER
label 'mem_max'
label 'long_running'
tag "${merge_group_id}"
Expand Down
2 changes: 1 addition & 1 deletion modules/bulk-salmon.nf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ process salmon {
}

process merge_bulk_quants {
container params.SCPCATOOLS_CONTAINER
container params.SCPCATOOLS_SLIM_CONTAINER
label 'mem_8'
publishDir "${params.results_dir}/${meta.project_id}", mode: 'copy'
tag "${meta.project_id}"
Expand Down
4 changes: 2 additions & 2 deletions modules/classify-celltypes.nf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ process classify_singler {


process classify_cellassign {
container params.SCPCATOOLS_CONTAINER
container params.SCPCATOOLS_SCVI_CONTAINER
publishDir (
path: "${meta.celltype_checkpoints_dir}",
mode: 'copy',
Expand Down Expand Up @@ -103,7 +103,7 @@ process classify_cellassign {
}

process add_celltypes_to_sce {
container params.SCPCATOOLS_CONTAINER
container params.SCPCATOOLS_SLIM_CONTAINER
label 'mem_4'
label 'cpus_2'
tag "${meta.library_id}"
Expand Down
2 changes: 1 addition & 1 deletion modules/cluster-sce.nf
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
// perform graph-based clustering on a processed SCE object
process cluster_sce {
container params.SCPCATOOLS_CONTAINER
container params.SCPCATOOLS_SLIM_CONTAINER
label 'mem_8'
tag "${meta.library_id}"
input:
Expand Down
2 changes: 1 addition & 1 deletion modules/export-anndata.nf
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@

// process for converting rds files containing an SCE to h5 containing anndata containing the RNA data
process export_anndata {
container params.SCPCATOOLS_CONTAINER
container params.SCPCATOOLS_ANNDATA_CONTAINER
label 'mem_16'
tag "${meta.library_id}"
publishDir "${params.results_dir}/${meta.project_id}/${meta.sample_id}", mode: 'copy'
Expand Down
2 changes: 1 addition & 1 deletion modules/qc-report.nf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
// generate QC report from unfiltered and filtered SCE.rds files using scpcaTools

process sce_qc_report {
container params.SCPCATOOLS_CONTAINER
container params.SCPCATOOLS_REPORTS_CONTAINER
label 'mem_16'
tag "${meta.library_id}"
publishDir "${params.results_dir}/${meta.project_id}/${meta.sample_id}", mode: 'copy'
Expand Down
12 changes: 6 additions & 6 deletions modules/sce-processing.nf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

// RNA only libraries
process make_unfiltered_sce {
container params.SCPCATOOLS_CONTAINER
container params.SCPCATOOLS_SLIM_CONTAINER
label 'mem_8'
tag "${meta.library_id}"
input:
Expand Down Expand Up @@ -48,7 +48,7 @@ process make_unfiltered_sce {
process make_merged_unfiltered_sce {
label 'mem_8'
tag "${rna_meta.library_id}"
container params.SCPCATOOLS_CONTAINER
container params.SCPCATOOLS_SLIM_CONTAINER
input:
tuple val(feature_meta), path(feature_alevin_dir),
val(rna_meta), path(alevin_dir),
Expand Down Expand Up @@ -105,7 +105,7 @@ process make_merged_unfiltered_sce {
}

process filter_sce {
container params.SCPCATOOLS_CONTAINER
container params.SCPCATOOLS_SLIM_CONTAINER
label 'mem_8'
tag "${meta.library_id}"
input:
Expand Down Expand Up @@ -138,7 +138,7 @@ process filter_sce {
}

process genetic_demux_sce {
container params.SCPCATOOLS_CONTAINER
container params.SCPCATOOLS_SLIM_CONTAINER
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I can't comment on it, but I think for process cellhash_demux_sce you need the Seurat container?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, I think I wasn't sure if that had everything we need, but we might as well give it a shot!

label 'mem_8'
tag "${meta.library_id}"
input:
Expand All @@ -164,7 +164,7 @@ process genetic_demux_sce {
}

process cellhash_demux_sce {
container params.SCPCATOOLS_CONTAINER
container params.SCPCATOOLS_SEURAT_CONTAINER
label 'mem_8'
tag "${meta.library_id}"
input:
Expand Down Expand Up @@ -192,7 +192,7 @@ process cellhash_demux_sce {
}

process post_process_sce {
container params.SCPCATOOLS_CONTAINER
container params.SCPCATOOLS_SLIM_CONTAINER
label 'mem_8'
tag "${meta.library_id}"
input:
Expand Down
2 changes: 1 addition & 1 deletion modules/spaceranger.nf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ process spaceranger {
}

process spaceranger_publish {
container params.SCPCATOOLS_CONTAINER
container params.SCPCATOOLS_SLIM_CONTAINER
tag "${meta.library_id}"
publishDir "${params.results_dir}/${meta.project_id}/${meta.sample_id}", mode: 'copy'
input:
Expand Down
24 changes: 22 additions & 2 deletions nextflow_schema.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -289,7 +289,27 @@
"properties": {
"SCPCATOOLS_CONTAINER": {
"type": "string",
"default": "ghcr.io/alexslemonade/scpca-tools:edge"
"default": "ghcr.io/alexslemonade/scpcatools:edge"
},
"SCPCATOOLS_SLIM_CONTAINER": {
"type": "string",
"default": "ghcr.io/alexslemonade/scpcatools-slim:edge"
},
"SCPCATOOLS_ANNDATA_CONTAINER": {
"type": "string",
"default": "ghcr.io/alexslemonade/scpcatools-anndata:edge"
},
"SCPCATOOLS_REPORTS_CONTAINER": {
"type": "string",
"default": "ghcr.io/alexslemonade/scpcatools-reports:edge"
},
"SCPCATOOLS_SEURAT_CONTAINER": {
"type": "string",
"default": "ghcr.io/alexslemonade/scpcatools-seurat:edge"
},
"SCPCATOOLS_SCVI_CONTAINER": {
"type": "string",
"default": "ghcr.io/alexslemonade/scpcatools-scvi:edge"
},
"ALEVINFRY_CONTAINER": {
"type": "string",
Expand Down Expand Up @@ -325,7 +345,7 @@
},
"TIDYVERSE_CONTAINER": {
"type": "string",
"default": "rocker/tidyverse:4.3.1"
"default": "rocker/tidyverse:4.4.0"
},
"VIREO_CONTAINER": {
"type": "string",
Expand Down
Loading