From b5d423b4d3334a10ac0091aaa9f65097f028ada9 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Fri, 3 Jan 2025 18:39:24 +0800 Subject: [PATCH 01/66] wip Signed-off-by: Paolo Di Tommaso --- .../nextflow/processor/TaskProcessor.groovy | 12 +- .../nextflow/processor/TaskTracker.groovy | 177 ++++++++++++++++++ .../nextflow/processor/TaskTrackerTest.groovy | 55 ++++++ 3 files changed, 242 insertions(+), 2 deletions(-) create mode 100644 modules/nextflow/src/main/groovy/nextflow/processor/TaskTracker.groovy create mode 100644 modules/nextflow/src/test/groovy/nextflow/processor/TaskTrackerTest.groovy diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy index 4b5fbf2791..9c0ef3ee96 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy @@ -583,7 +583,7 @@ class TaskProcessor { this.openPorts = createPortsArray(opInputs.size()) config.getOutputs().setSingleton(singleton) def interceptor = new TaskProcessorInterceptor(opInputs, singleton) - def params = [inputs: opInputs, maxForks: session.poolSize, listeners: [interceptor] ] + def params = [inputs: opInputs, outputs: config.getOutputs().getChannels(), maxForks: session.poolSize, listeners: [interceptor, new TaskTracker()] ] def invoke = new InvokeTaskAdapter(this, opInputs.size()) session.allOperators << (operator = new DataflowOperator(group, params, invoke)) @@ -1519,10 +1519,18 @@ class TaskProcessor { // and result in a potential error. See https://github.com/nextflow-io/nextflow/issues/3768 final copy = x instanceof List && x instanceof Cloneable ? x.clone() : x // emit the final value - ch.bind(copy) + bindOutParam0(param, ch, copy) } } + protected void bindOutParam0(OutParam params, DataflowWriteChannel ch, Object value) { + final index = operator.getOutputs().indexOf(ch) + if( index==-1 ) + throw new IllegalArgumentException("Unable to determine index of output channel for param: $params") + // use the operator "bindOutput" to rely on underlying binding event APIs + operator.bindOutput(index, value) + } + protected void collectOutputs( TaskRun task ) { collectOutputs( task, task.getTargetDir(), task.@stdout, task.context ) } diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskTracker.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskTracker.groovy new file mode 100644 index 0000000000..ecd73f7fe1 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskTracker.groovy @@ -0,0 +1,177 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.processor + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicInteger + +import groovy.transform.Canonical +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import groovyx.gpars.dataflow.DataflowWriteChannel +import groovyx.gpars.dataflow.operator.DataflowEventAdapter +import groovyx.gpars.dataflow.operator.DataflowProcessor +/** + * + * @author Paolo Di Tommaso + */ +@Slf4j +@CompileStatic +class TaskTracker extends DataflowEventAdapter { + + static class NullMessage { } + + @Canonical + static class RunEntry { + int runId + /** Both process and operator runs have associated a {@link DataflowProcessor} */ + DataflowProcessor processor + /** The corresponding {@link TaskStartParams} in the case process run */ + TaskStartParams task + /** The identities of the upstream tasks */ + Set upstreamIds + } + + private static final Map messages = new ConcurrentHashMap<>() + + private static final ThreadLocal entry = new ThreadLocal<>() + + private static final AtomicInteger generator = new AtomicInteger() + + private static final Map runs = new ConcurrentHashMap<>() + + static int len() { + return messages.size() + } + + static void clean() { + messages.clear() + generator.set(0) + } + + @Override + List beforeRun(DataflowProcessor processor, List messages) { + assert messages.size() == 2, "Unexpected processes input messages size=${messages.size()}" + assert messages[0] instanceof TaskStartParams + assert messages[1] instanceof List + + final task = getTaskParams(messages) + final entry = runs.get(task.id) + if( entry==null ) { + // nothing to do + // this is expected for the first task that has not yet emitted any output message + return messages + } + // find the upstream tasks id + final upstream = findUpstreamTasks(messages) + // the second entry of messages list represent the task inputs list + // apply the de-normalization before returning it + final inputs = messages[1] as List + messages[1] = denormalizeMessages(inputs) + log.debug "Input messages ${inputs.collect(it-> str(it)).join(', ')}\n^^ Upstream tasks: ${upstream.join(',')}" + return messages + } + + protected String str(Object o) { + return o!=null ? "${o.class.name}@${System.identityHashCode(o)}" : null + } + + protected Set findUpstreamTasks(final int msgId, Set upstream = new HashSet<>(10)) { + final entry = messages.get(msgId) + if( entry==null ) { + return upstream + } + if( entry.task ) { + upstream.add(entry.task.id) + return upstream + } + return upstream + } + + protected Set findUpstreamTasks(List messages) { + // find upstream tasks and restore nulls + final result = new HashSet() + for( Object msg : messages ) { + if( msg==null ) + throw new IllegalArgumentException("Message cannot be a null object") + final msgId = System.identityHashCode(msg) + result.addAll(findUpstreamTasks(msgId)) + } + return result + } + + @Override + Object messageSentOut(DataflowProcessor processor, DataflowWriteChannel channel, int index, Object message) { + // normalize the message + final msg = normalizeMessage(message) + // determine the current run entry + final entry = getOrCreateEntry(processor) + // map the message with the run entry where it has been output + messages.put(System.identityHashCode(msg), entry) + return msg + } + + protected RunEntry getOrCreateEntry(DataflowProcessor processor) { + def entry = entry.get() + if( entry==null ) { + entry = new RunEntry(generator.incrementAndGet(), processor) + TaskTracker.entry.set(entry) + } + return entry + } + + protected Object normalizeMessage(Object message) { + // map a "null" value into an instance of "NullMessage" + // because it's needed the object identity to track the message flow + return message!=null ? message : new NullMessage() + } + + protected Object denormalizeMessage(Object msg) { + return msg !instanceof NullMessage ? msg : null + } + + protected List denormalizeMessages(List messages) { + return messages.collect(it-> denormalizeMessage(it)) + } + + protected TaskStartParams getTaskParams(List messages) { + if( messages && messages[0] instanceof TaskStartParams ) + return messages[0] as TaskStartParams + else + throw new IllegalArgumentException("Unable to find required TaskStartParams object - offending messages=$messages") + } + + /** + * track all outputs send out + */ + @Override + void afterRun(DataflowProcessor processor, List inputs) { + final entry = entry.get() + assert entry!=null, "Missing run entry for process=$processor" + try { + // update the entry with the task param + entry.task = getTaskParams(inputs) + // map the run entry with the task Id + runs.put(entry.task.id, entry) + } + finally { + // cleanup the current run entry + TaskTracker.entry.remove() + } + } +} diff --git a/modules/nextflow/src/test/groovy/nextflow/processor/TaskTrackerTest.groovy b/modules/nextflow/src/test/groovy/nextflow/processor/TaskTrackerTest.groovy new file mode 100644 index 0000000000..2234d50ed7 --- /dev/null +++ b/modules/nextflow/src/test/groovy/nextflow/processor/TaskTrackerTest.groovy @@ -0,0 +1,55 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.processor + +import test.Dsl2Spec +import test.MockScriptRunner + +/** + * + * @author Paolo Di Tommaso + */ +class TaskTrackerTest extends Dsl2Spec { + + def 'should define a process with output alias' () { + given: + def SCRIPT = ''' + + process foo { + output: val x, emit: 'ch1' + exec: x = 1 + } + + process bar { + input: val x + exec: return x*x + } + + workflow { + foo | map { x-> x+1 } | bar + } + ''' + + when: + def runner = new MockScriptRunner() + def result = runner.setScript(SCRIPT).execute() + then: + noExceptionThrown() + } + +} From ca9fe882cae93b7edf043ea3734d5314101d8970 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sat, 4 Jan 2025 09:43:05 +0800 Subject: [PATCH 02/66] Task provenance - poc #1 Signed-off-by: Paolo Di Tommaso --- .../src/main/groovy/nextflow/Session.groovy | 8 + .../executor/BashWrapperBuilder.groovy | 11 +- .../groovy/nextflow/processor/TaskBean.groovy | 6 + .../nextflow/processor/TaskProcessor.groovy | 31 ++- .../groovy/nextflow/processor/TaskRun.groovy | 5 + .../nextflow/processor/TaskTracker.groovy | 177 ------------------ .../nextflow/provenance/ProvTracker.groovy | 92 +++++++++ .../nextflow/processor/TaskTrackerTest.groovy | 55 ------ .../provenance/ProvTrackerTest.groovy | 94 ++++++++++ 9 files changed, 227 insertions(+), 252 deletions(-) delete mode 100644 modules/nextflow/src/main/groovy/nextflow/processor/TaskTracker.groovy create mode 100644 modules/nextflow/src/main/groovy/nextflow/provenance/ProvTracker.groovy delete mode 100644 modules/nextflow/src/test/groovy/nextflow/processor/TaskTrackerTest.groovy create mode 100644 modules/nextflow/src/test/groovy/nextflow/provenance/ProvTrackerTest.groovy diff --git a/modules/nextflow/src/main/groovy/nextflow/Session.groovy b/modules/nextflow/src/main/groovy/nextflow/Session.groovy index f394245259..5d91cfe6a6 100644 --- a/modules/nextflow/src/main/groovy/nextflow/Session.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/Session.groovy @@ -53,6 +53,7 @@ import nextflow.processor.ErrorStrategy import nextflow.processor.TaskFault import nextflow.processor.TaskHandler import nextflow.processor.TaskProcessor +import nextflow.provenance.ProvTracker import nextflow.script.BaseScript import nextflow.script.ProcessConfig import nextflow.script.ProcessFactory @@ -223,6 +224,8 @@ class Session implements ISession { private DAG dag + private ProvTracker provenance + private CacheDB cache private Barrier processesBarrier = new Barrier() @@ -381,6 +384,9 @@ class Session implements ISession { // -- DAG object this.dag = new DAG() + // -- create the provenance tracker + this.provenance = new ProvTracker() + // -- init output dir this.outputDir = FileHelper.toCanonicalPath(config.outputDir ?: 'results') @@ -856,6 +862,8 @@ class Session implements ISession { DAG getDag() { this.dag } + ProvTracker getProvenance() { provenance } + ExecutorService getExecService() { execService } /** diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/BashWrapperBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/BashWrapperBuilder.groovy index 6637d7a9b7..e5e771f5c5 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/BashWrapperBuilder.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/BashWrapperBuilder.groovy @@ -85,10 +85,8 @@ class BashWrapperBuilder { log.warn "Invalid value for `NXF_DEBUG` variable: $str -- See http://www.nextflow.io/docs/latest/config.html#environment-variables" } BASH = Collections.unmodifiableList( level > 0 ? ['/bin/bash','-uex'] : ['/bin/bash','-ue'] ) - } - @Delegate ScriptFileCopyStrategy copyStrategy @@ -480,6 +478,7 @@ class BashWrapperBuilder { protected String getTaskMetadata() { final lines = new StringBuilder() lines << '### ---\n' + lines << "### id: '${bean.taskId}'\n" lines << "### name: '${bean.name}'\n" if( bean.arrayIndexName ) { lines << '### array:\n' @@ -493,12 +492,18 @@ class BashWrapperBuilder { if( containerConfig?.isEnabled() ) lines << "### container: '${bean.containerImage}'\n" - if( outputFiles.size() > 0 ) { + if( outputFiles ) { lines << '### outputs:\n' for( final output : bean.outputFiles ) lines << "### - '${output}'\n" } + if( bean.upstreamTasks ) { + lines << '### upstream-tasks:\n' + for( final it : bean.upstreamTasks ) + lines << "### - '${it}'\n" + } + lines << '### ...\n' } diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskBean.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskBean.groovy index 5d4175aeff..9c5698d611 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskBean.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskBean.groovy @@ -36,6 +36,10 @@ class TaskBean implements Serializable, Cloneable { String name + TaskId taskId + + Set upstreamTasks + def input def scratch @@ -122,6 +126,8 @@ class TaskBean implements Serializable, Cloneable { TaskBean(TaskRun task) { this.name = task.name + this.taskId = task.id + this.upstreamTasks = task.upstreamTasks // set the input (when available) this.input = task.stdin diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy index 9c0ef3ee96..e0698be111 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy @@ -15,6 +15,7 @@ */ package nextflow.processor +import nextflow.provenance.ProvTracker import nextflow.trace.TraceRecord import static nextflow.processor.ErrorStrategy.* @@ -259,6 +260,8 @@ class TaskProcessor { private TaskArrayCollector arrayCollector + private ProvTracker provenance + private CompilerConfiguration compilerConfig() { final config = new CompilerConfiguration() config.addCompilationCustomizers( new ASTTransformationCustomizer(TaskTemplateVarsXform) ) @@ -318,6 +321,7 @@ class TaskProcessor { final arraySize = config.getArray() this.arrayCollector = arraySize > 0 ? new TaskArrayCollector(this, executor, arraySize) : null + this.provenance = session.getProvenance() } /** @@ -583,7 +587,7 @@ class TaskProcessor { this.openPorts = createPortsArray(opInputs.size()) config.getOutputs().setSingleton(singleton) def interceptor = new TaskProcessorInterceptor(opInputs, singleton) - def params = [inputs: opInputs, outputs: config.getOutputs().getChannels(), maxForks: session.poolSize, listeners: [interceptor, new TaskTracker()] ] + def params = [inputs: opInputs, maxForks: session.poolSize, listeners: [interceptor] ] def invoke = new InvokeTaskAdapter(this, opInputs.size()) session.allOperators << (operator = new DataflowOperator(group, params, invoke)) @@ -623,15 +627,16 @@ class TaskProcessor { final protected void invokeTask( Object[] args ) { assert args.size()==2 final params = (TaskStartParams) args[0] - final values = (List) args[1] - + final inputs = (List) args[1] // create and initialize the task instance to be executed - log.trace "Invoking task > $name with params=$params; values=$values" + log.trace "Invoking task > $name with params=$params; values=${inputs}" // -- create the task run instance final task = createTaskRun(params) // -- set the task instance as the current in this thread currentTask.set(task) + // track the task provenance for the given inputs + final values = provenance.beforeRun(task, inputs) // -- validate input lengths validateInputTuples(values) @@ -1461,7 +1466,7 @@ class TaskProcessor { fairBindOutputs0(tuples, task) } else { - bindOutputs0(tuples) + bindOutputs0(tuples, task) } // -- finally prints out the task output when 'debug' is true @@ -1492,7 +1497,7 @@ class TaskProcessor { } } - protected void bindOutputs0(Map tuples) { + protected void bindOutputs0(Map tuples, TaskRun task) { // -- bind out the collected values for( OutParam param : config.getOutputs() ) { final outValue = tuples[param.index] @@ -1505,11 +1510,11 @@ class TaskProcessor { } log.trace "Process $name > Binding out param: ${param} = ${outValue}" - bindOutParam(param, outValue) + bindOutParam(param, outValue, task) } } - protected void bindOutParam( OutParam param, List values ) { + protected void bindOutParam( OutParam param, List values, TaskRun task ) { log.trace "<$name> Binding param $param with $values" final x = values.size() == 1 ? values[0] : values final ch = param.getOutChannel() @@ -1519,18 +1524,10 @@ class TaskProcessor { // and result in a potential error. See https://github.com/nextflow-io/nextflow/issues/3768 final copy = x instanceof List && x instanceof Cloneable ? x.clone() : x // emit the final value - bindOutParam0(param, ch, copy) + provenance.bindOutput(task, ch, copy) } } - protected void bindOutParam0(OutParam params, DataflowWriteChannel ch, Object value) { - final index = operator.getOutputs().indexOf(ch) - if( index==-1 ) - throw new IllegalArgumentException("Unable to determine index of output channel for param: $params") - // use the operator "bindOutput" to rely on underlying binding event APIs - operator.bindOutput(index, value) - } - protected void collectOutputs( TaskRun task ) { collectOutputs( task, task.getTargetDir(), task.@stdout, task.context ) } diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy index 38b4ba4782..49851e035c 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy @@ -98,6 +98,11 @@ class TaskRun implements Cloneable { */ Map outputs = [:] + /** + * Holds the IDs of the upstream tasks that contributed to trigger + * the execution of this task run + */ + Set upstreamTasks void setInput( InParam param, Object value = null ) { assert param diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskTracker.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskTracker.groovy deleted file mode 100644 index ecd73f7fe1..0000000000 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskTracker.groovy +++ /dev/null @@ -1,177 +0,0 @@ -/* - * Copyright 2013-2024, Seqera Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package nextflow.processor - -import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.atomic.AtomicInteger - -import groovy.transform.Canonical -import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j -import groovyx.gpars.dataflow.DataflowWriteChannel -import groovyx.gpars.dataflow.operator.DataflowEventAdapter -import groovyx.gpars.dataflow.operator.DataflowProcessor -/** - * - * @author Paolo Di Tommaso - */ -@Slf4j -@CompileStatic -class TaskTracker extends DataflowEventAdapter { - - static class NullMessage { } - - @Canonical - static class RunEntry { - int runId - /** Both process and operator runs have associated a {@link DataflowProcessor} */ - DataflowProcessor processor - /** The corresponding {@link TaskStartParams} in the case process run */ - TaskStartParams task - /** The identities of the upstream tasks */ - Set upstreamIds - } - - private static final Map messages = new ConcurrentHashMap<>() - - private static final ThreadLocal entry = new ThreadLocal<>() - - private static final AtomicInteger generator = new AtomicInteger() - - private static final Map runs = new ConcurrentHashMap<>() - - static int len() { - return messages.size() - } - - static void clean() { - messages.clear() - generator.set(0) - } - - @Override - List beforeRun(DataflowProcessor processor, List messages) { - assert messages.size() == 2, "Unexpected processes input messages size=${messages.size()}" - assert messages[0] instanceof TaskStartParams - assert messages[1] instanceof List - - final task = getTaskParams(messages) - final entry = runs.get(task.id) - if( entry==null ) { - // nothing to do - // this is expected for the first task that has not yet emitted any output message - return messages - } - // find the upstream tasks id - final upstream = findUpstreamTasks(messages) - // the second entry of messages list represent the task inputs list - // apply the de-normalization before returning it - final inputs = messages[1] as List - messages[1] = denormalizeMessages(inputs) - log.debug "Input messages ${inputs.collect(it-> str(it)).join(', ')}\n^^ Upstream tasks: ${upstream.join(',')}" - return messages - } - - protected String str(Object o) { - return o!=null ? "${o.class.name}@${System.identityHashCode(o)}" : null - } - - protected Set findUpstreamTasks(final int msgId, Set upstream = new HashSet<>(10)) { - final entry = messages.get(msgId) - if( entry==null ) { - return upstream - } - if( entry.task ) { - upstream.add(entry.task.id) - return upstream - } - return upstream - } - - protected Set findUpstreamTasks(List messages) { - // find upstream tasks and restore nulls - final result = new HashSet() - for( Object msg : messages ) { - if( msg==null ) - throw new IllegalArgumentException("Message cannot be a null object") - final msgId = System.identityHashCode(msg) - result.addAll(findUpstreamTasks(msgId)) - } - return result - } - - @Override - Object messageSentOut(DataflowProcessor processor, DataflowWriteChannel channel, int index, Object message) { - // normalize the message - final msg = normalizeMessage(message) - // determine the current run entry - final entry = getOrCreateEntry(processor) - // map the message with the run entry where it has been output - messages.put(System.identityHashCode(msg), entry) - return msg - } - - protected RunEntry getOrCreateEntry(DataflowProcessor processor) { - def entry = entry.get() - if( entry==null ) { - entry = new RunEntry(generator.incrementAndGet(), processor) - TaskTracker.entry.set(entry) - } - return entry - } - - protected Object normalizeMessage(Object message) { - // map a "null" value into an instance of "NullMessage" - // because it's needed the object identity to track the message flow - return message!=null ? message : new NullMessage() - } - - protected Object denormalizeMessage(Object msg) { - return msg !instanceof NullMessage ? msg : null - } - - protected List denormalizeMessages(List messages) { - return messages.collect(it-> denormalizeMessage(it)) - } - - protected TaskStartParams getTaskParams(List messages) { - if( messages && messages[0] instanceof TaskStartParams ) - return messages[0] as TaskStartParams - else - throw new IllegalArgumentException("Unable to find required TaskStartParams object - offending messages=$messages") - } - - /** - * track all outputs send out - */ - @Override - void afterRun(DataflowProcessor processor, List inputs) { - final entry = entry.get() - assert entry!=null, "Missing run entry for process=$processor" - try { - // update the entry with the task param - entry.task = getTaskParams(inputs) - // map the run entry with the task Id - runs.put(entry.task.id, entry) - } - finally { - // cleanup the current run entry - TaskTracker.entry.remove() - } - } -} diff --git a/modules/nextflow/src/main/groovy/nextflow/provenance/ProvTracker.groovy b/modules/nextflow/src/main/groovy/nextflow/provenance/ProvTracker.groovy new file mode 100644 index 0000000000..5eebc2edb1 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/provenance/ProvTracker.groovy @@ -0,0 +1,92 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.provenance + +import java.util.concurrent.ConcurrentHashMap + +import groovy.transform.CompileStatic +import groovyx.gpars.dataflow.DataflowWriteChannel +import nextflow.processor.TaskId +import nextflow.processor.TaskRun +/** + * + * @author Paolo Di Tommaso + */ +@CompileStatic +class ProvTracker { + + static class NullMessage { } + + private Map messages = new ConcurrentHashMap<>() + + List beforeRun(TaskRun task, List messages) { + // find the upstream tasks id + findUpstreamTasks(task, messages) + // the second entry of messages list represent the task inputs list + // apply the de-normalization before returning it + return denormalizeMessages(messages) + } + + protected void findUpstreamTasks(TaskRun task, List messages) { + // find upstream tasks and restore nulls + final result = new HashSet() + for( Object msg : messages ) { + if( msg==null ) + throw new IllegalArgumentException("Message cannot be a null object") + final msgId = System.identityHashCode(msg) + result.addAll(findUpstreamTasks0(msgId,result)) + } + // finally bind the result to the task record + task.upstreamTasks = result + } + + protected Set findUpstreamTasks0(final int msgId, Set upstream) { + final task = messages.get(msgId) + if( task==null ) { + return upstream + } + if( task ) { + upstream.add(task.id) + return upstream + } + return upstream + } + + protected Object denormalizeMessage(Object msg) { + return msg !instanceof NullMessage ? msg : null + } + + protected List denormalizeMessages(List messages) { + return messages.collect(it-> denormalizeMessage(it)) + } + + protected Object normalizeMessage(Object message) { + // map a "null" value into an instance of "NullMessage" + // because it's needed the object identity to track the message flow + return message!=null ? message : new NullMessage() + } + + void bindOutput(TaskRun task, DataflowWriteChannel ch, Object msg) { + final value = normalizeMessage(msg) + // map the message with the run where it has been output + messages.put(System.identityHashCode(value), task) + // now emit the value + ch.bind(value) + } + +} diff --git a/modules/nextflow/src/test/groovy/nextflow/processor/TaskTrackerTest.groovy b/modules/nextflow/src/test/groovy/nextflow/processor/TaskTrackerTest.groovy deleted file mode 100644 index 2234d50ed7..0000000000 --- a/modules/nextflow/src/test/groovy/nextflow/processor/TaskTrackerTest.groovy +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2013-2024, Seqera Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package nextflow.processor - -import test.Dsl2Spec -import test.MockScriptRunner - -/** - * - * @author Paolo Di Tommaso - */ -class TaskTrackerTest extends Dsl2Spec { - - def 'should define a process with output alias' () { - given: - def SCRIPT = ''' - - process foo { - output: val x, emit: 'ch1' - exec: x = 1 - } - - process bar { - input: val x - exec: return x*x - } - - workflow { - foo | map { x-> x+1 } | bar - } - ''' - - when: - def runner = new MockScriptRunner() - def result = runner.setScript(SCRIPT).execute() - then: - noExceptionThrown() - } - -} diff --git a/modules/nextflow/src/test/groovy/nextflow/provenance/ProvTrackerTest.groovy b/modules/nextflow/src/test/groovy/nextflow/provenance/ProvTrackerTest.groovy new file mode 100644 index 0000000000..571e127b8c --- /dev/null +++ b/modules/nextflow/src/test/groovy/nextflow/provenance/ProvTrackerTest.groovy @@ -0,0 +1,94 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.provenance + +import groovyx.gpars.dataflow.DataflowQueue +import groovyx.gpars.dataflow.DataflowWriteChannel +import nextflow.processor.TaskId +import nextflow.processor.TaskRun +import spock.lang.Specification + +/** + * + * @author Paolo Di Tommaso + */ +class ProvTrackerTest extends Specification { + + def 'should normalize null values' () { + given: + def prov = new ProvTracker() + and: + def t1 = new TaskRun(id: new TaskId(1)) + def t2 = new TaskRun(id: new TaskId(2)) + and: + def msg1 = ['foo'] + def msg2 = ['foo', new ProvTracker.NullMessage()] + + when: + def result1 = prov.beforeRun(t1, msg1) + then: + result1 == msg1 + and: + t1.upstreamTasks == [] as Set + + when: + def result2 = prov.beforeRun(t2, msg2) + then: + result2 == ['foo', null] + and: + t2.upstreamTasks == [] as Set + } + + def 'should bind value to task run' () { + given: + def prov = new ProvTracker() + and: + def t1 = new TaskRun(id: new TaskId(1)) + def c1 = new DataflowQueue() + def v1 = 'foo' + + when: + prov.bindOutput(t1, c1, v1) + then: + c1.val == 'foo' + and: + prov.@messages.get(System.identityHashCode(v1)) == t1 + } + + def 'should determine upstream tasks' () { + given: + def prov = new ProvTracker() + and: + def t1 = new TaskRun(id: new TaskId(1)) + def t2 = new TaskRun(id: new TaskId(2)) + def t3 = new TaskRun(id: new TaskId(3)) + and: + def v1 = new Object() + def v2 = new Object() + def v3 = new Object() + + when: + prov.bindOutput(t1, Mock(DataflowWriteChannel), v1) + prov.bindOutput(t2, Mock(DataflowWriteChannel), v2) + and: + prov.beforeRun(t3, [v1, v2]) + then: + t3.upstreamTasks == [t1.id, t2.id] as Set + } + +} From 494bbb0ca1274241067117668c4e534ee4b4e5a9 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sun, 5 Jan 2025 20:09:18 +0800 Subject: [PATCH 03/66] Task provenance - poc #2 Signed-off-by: Paolo Di Tommaso --- .../src/main/groovy/nextflow/Session.groovy | 9 +- .../groovy/nextflow/extension/BranchOp.groovy | 2 +- .../nextflow/extension/CollectFileOp.groovy | 12 +- .../nextflow/extension/CollectOp.groovy | 7 +- .../groovy/nextflow/extension/ConcatOp.groovy | 4 +- .../nextflow/extension/DataflowHelper.groovy | 132 ++++++++--- .../nextflow/extension/GroupTupleOp.groovy | 2 +- .../groovy/nextflow/extension/MapOp.groovy | 9 +- .../main/groovy/nextflow/extension/Op.groovy | 166 ++++++++++++++ .../nextflow/extension/OperatorImpl.groovy | 15 +- .../groovy/nextflow/processor/TaskId.groovy | 6 + .../nextflow/processor/TaskProcessor.groovy | 36 +-- .../groovy/nextflow/processor/TaskRun.groovy | 7 +- .../groovy/nextflow/prov/OperatorRun.groovy | 40 ++++ .../src/main/groovy/nextflow/prov/Prov.groovy | 46 ++++ .../main/groovy/nextflow/prov/Tracker.groovy | 162 ++++++++++++++ .../main/groovy/nextflow/prov/TrailRun.groovy | 25 +++ .../nextflow/provenance/ProvTracker.groovy | 92 -------- .../nextflow/extension/ConcatOpTest.groovy | 2 +- .../groovy/nextflow/extension/OpTest.groovy | 47 ++++ .../nextflow/extension/UntilManyOpTest.groovy | 4 +- .../test/groovy/nextflow/prov/ProvTest.groovy | 206 ++++++++++++++++++ .../groovy/nextflow/prov/TrackerTest.groovy | 123 +++++++++++ .../provenance/ProvTrackerTest.groovy | 94 -------- .../testFixtures/groovy/test/Dsl2Spec.groovy | 5 +- .../groovy/test/MockHelpers.groovy | 2 +- .../groovy/test/TestHelper.groovy | 36 +++ 27 files changed, 1024 insertions(+), 267 deletions(-) create mode 100644 modules/nextflow/src/main/groovy/nextflow/extension/Op.groovy create mode 100644 modules/nextflow/src/main/groovy/nextflow/prov/OperatorRun.groovy create mode 100644 modules/nextflow/src/main/groovy/nextflow/prov/Prov.groovy create mode 100644 modules/nextflow/src/main/groovy/nextflow/prov/Tracker.groovy create mode 100644 modules/nextflow/src/main/groovy/nextflow/prov/TrailRun.groovy delete mode 100644 modules/nextflow/src/main/groovy/nextflow/provenance/ProvTracker.groovy create mode 100644 modules/nextflow/src/test/groovy/nextflow/extension/OpTest.groovy create mode 100644 modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy create mode 100644 modules/nextflow/src/test/groovy/nextflow/prov/TrackerTest.groovy delete mode 100644 modules/nextflow/src/test/groovy/nextflow/provenance/ProvTrackerTest.groovy diff --git a/modules/nextflow/src/main/groovy/nextflow/Session.groovy b/modules/nextflow/src/main/groovy/nextflow/Session.groovy index 5d91cfe6a6..5e4c3dac0a 100644 --- a/modules/nextflow/src/main/groovy/nextflow/Session.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/Session.groovy @@ -53,7 +53,7 @@ import nextflow.processor.ErrorStrategy import nextflow.processor.TaskFault import nextflow.processor.TaskHandler import nextflow.processor.TaskProcessor -import nextflow.provenance.ProvTracker +import nextflow.prov.Tracker import nextflow.script.BaseScript import nextflow.script.ProcessConfig import nextflow.script.ProcessFactory @@ -224,8 +224,6 @@ class Session implements ISession { private DAG dag - private ProvTracker provenance - private CacheDB cache private Barrier processesBarrier = new Barrier() @@ -384,9 +382,6 @@ class Session implements ISession { // -- DAG object this.dag = new DAG() - // -- create the provenance tracker - this.provenance = new ProvTracker() - // -- init output dir this.outputDir = FileHelper.toCanonicalPath(config.outputDir ?: 'results') @@ -862,7 +857,7 @@ class Session implements ISession { DAG getDag() { this.dag } - ProvTracker getProvenance() { provenance } + Tracker getProvenance() { provenance } ExecutorService getExecService() { execService } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/BranchOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/BranchOp.groovy index 7fc5067620..868db7131f 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/BranchOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/BranchOp.groovy @@ -53,7 +53,7 @@ class BranchOp { protected void doNext(it) { TokenBranchChoice ret = switchDef.closure.call(it) if( ret ) { - targets[ret.choice].bind(ret.value) + Op.bind(targets[ret.choice], ret.value) } } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/CollectFileOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/CollectFileOp.groovy index 6f3dcd2b75..4b8c181d50 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/CollectFileOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/CollectFileOp.groovy @@ -16,6 +16,9 @@ package nextflow.extension +import static nextflow.util.CacheHelper.* +import static nextflow.util.CheckHelper.* + import java.nio.file.Path import groovy.util.logging.Slf4j @@ -28,8 +31,6 @@ import nextflow.file.FileHelper import nextflow.file.SimpleFileCollector import nextflow.file.SortFileCollector import nextflow.util.CacheHelper -import static nextflow.util.CacheHelper.HashMode -import static nextflow.util.CheckHelper.checkParams /** * Implements the body of {@link OperatorImpl#collectFile(groovyx.gpars.dataflow.DataflowReadChannel)} operator * @@ -185,10 +186,10 @@ class CollectFileOp { protected emitItems( obj ) { // emit collected files to 'result' channel collector.saveTo(storeDir).each { - result.bind(it) + Op.bind(result,it) } // close the channel - result.bind(Channel.STOP) + Op.bind(result,Channel.STOP) // close the collector collector.safeClose() } @@ -261,9 +262,8 @@ class CollectFileOp { return collector } - DataflowWriteChannel apply() { - DataflowHelper.subscribeImpl( channel, [onNext: this.&processItem, onComplete: this.&emitItems] ) + DataflowHelper.subscribeImpl( channel, true, [onNext: this.&processItem, onComplete: this.&emitItems] ) return result } } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/CollectOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/CollectOp.groovy index 9d7b02558f..71cdf5aedb 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/CollectOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/CollectOp.groovy @@ -16,7 +16,7 @@ package nextflow.extension -import static nextflow.util.CheckHelper.checkParams +import static nextflow.util.CheckHelper.* import groovy.transform.CompileStatic import groovyx.gpars.dataflow.DataflowReadChannel @@ -55,7 +55,10 @@ class CollectOp { Map events = [:] events.onNext = { append(result, it) } - events.onComplete = { target << ( result ? new ArrayBag(normalise(result)) : Channel.STOP ) } + events.onComplete = { + final msg = result ? new ArrayBag(normalise(result)) : Channel.STOP + Op.bind(target, msg) + } DataflowHelper.subscribeImpl(source, events) return target diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/ConcatOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/ConcatOp.groovy index f21a34d76b..34e9ceca38 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/ConcatOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/ConcatOp.groovy @@ -56,10 +56,10 @@ class ConcatOp { def next = index < channels.size() ? channels[index] : null def events = new HashMap(2) - events.onNext = { result.bind(it) } + events.onNext = { Op.bind(result, it) } events.onComplete = { if(next) append(result, channels, index) - else result.bind(Channel.STOP) + else Op.bind(result, Channel.STOP) } DataflowHelper.subscribeImpl(current, events) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy index 14c775b04e..4543a89710 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy @@ -36,7 +36,6 @@ import nextflow.Channel import nextflow.Global import nextflow.Session import nextflow.dag.NodeMarker -import static java.util.Arrays.asList /** * This class provides helper methods to implement nextflow operators * @@ -45,6 +44,70 @@ import static java.util.Arrays.asList @Slf4j class DataflowHelper { + static class OpParams { + List inputs + List outputs + List listeners + boolean accumulator + + OpParams() { } + + OpParams(Map params) { + this.inputs = params.inputs as List ?: List.of() + this.outputs = params.outputs as List ?: List.of() + this.listeners = params.listeners as List ?: List.of() + } + + OpParams withInput(DataflowReadChannel channel) { + assert channel != null + this.inputs = List.of(channel) + return this + } + + OpParams withInputs(List channels) { + assert channels != null + this.inputs = channels + return this + } + + OpParams withOutput(DataflowWriteChannel channel) { + assert channel != null + this.outputs = List.of(channel) + return this + } + + OpParams withOutputs(List channels) { + assert channels != null + this.outputs = channels + return this + } + + OpParams withListener(DataflowEventListener listener) { + assert listener != null + this.listeners = List.of(listener) + return this + } + + OpParams withListeners(List listeners) { + assert listeners != null + this.listeners = listeners + return this + } + + OpParams withAccumulator(boolean acc) { + this.accumulator = acc + return this + } + + Map toMap() { + final ret = new HashMap() + ret.inputs = inputs ?: List.of() + ret.outputs = outputs ?: List.of() + ret.listeners = listeners ?: List.of() + return ret + } + } + private static Session getSession() { Global.getSession() as Session } /** @@ -141,6 +204,7 @@ class DataflowHelper { * @param params The map holding inputs, outputs channels and other parameters * @param code The closure to be executed by the operator */ + @Deprecated static DataflowProcessor newOperator( Map params, Closure code ) { // -- add a default error listener @@ -149,13 +213,13 @@ class DataflowHelper { params.listeners = [ DEF_ERROR_LISTENER ] } - final op = Dataflow.operator(params, code) - NodeMarker.appendOperator(op) - if( session && session.allOperators != null ) { - session.allOperators.add(op) - } + return newOperator0(new OpParams(params), code) + } - return op + static DataflowProcessor newOperator( OpParams params, Closure code ) { + if( !params.listeners ) + params.withListener(DEF_ERROR_LISTENER) + return newOperator0(params, code) } /** @@ -195,16 +259,25 @@ class DataflowHelper { * @param code The closure to be executed by the operator */ static DataflowProcessor newOperator( DataflowReadChannel input, DataflowWriteChannel output, DataflowEventListener listener, Closure code ) { - if( !listener ) listener = DEF_ERROR_LISTENER - def params = [:] + final params = [:] params.inputs = [input] params.outputs = [output] params.listeners = [listener] - final op = Dataflow.operator(params, code) + return newOperator0(new OpParams(params), code) + } + + static private DataflowProcessor newOperator0(OpParams params, Closure code) { + assert params + assert params.inputs + assert params.listeners + + // create the underlying dataflow operator + final op = Dataflow.operator(params.toMap(), Op.instrument(code, params.accumulator)) + // track the operator as dag node NodeMarker.appendOperator(op) if( session && session.allOperators != null ) { session.allOperators << op @@ -236,14 +309,11 @@ class DataflowHelper { } - /** - * Subscribe *onNext*, *onError* and *onComplete* - * - * @param source - * @param closure - * @return - */ static final DataflowProcessor subscribeImpl(final DataflowReadChannel source, final Map events ) { + subscribeImpl(source, false, events) + } + + static final DataflowProcessor subscribeImpl(final DataflowReadChannel source, final boolean accumulator, final Map events ) { checkSubscribeHandlers(events) def error = false @@ -276,13 +346,12 @@ class DataflowHelper { } } + final params = new OpParams() + .withInput(source) + .withListener(listener) + .withAccumulator(accumulator) - final Map parameters = new HashMap(); - parameters.put("inputs", [source]) - parameters.put("outputs", []) - parameters.put('listeners', [listener]) - - newOperator (parameters) { + newOperator (params) { if( events.onNext ) { events.onNext.call(it) } @@ -292,7 +361,7 @@ class DataflowHelper { } } - + @Deprecated static DataflowProcessor chainImpl(final DataflowReadChannel source, final DataflowWriteChannel target, final Map params, final Closure closure) { final Map parameters = new HashMap(params) @@ -302,6 +371,10 @@ class DataflowHelper { newOperator(parameters, new ChainWithClosure(closure)) } + static DataflowProcessor chainImpl(OpParams params, final Closure closure) { + newOperator(params, new ChainWithClosure(closure)) + } + /** * Implements the {@code #reduce} operator * @@ -321,7 +394,7 @@ class DataflowHelper { * call the passed closure each time */ void afterRun(final DataflowProcessor processor, final List messages) { - final item = messages.get(0) + final item = Op.unwrap(messages).get(0) final value = accum == null ? item : closure.call(accum, item) if( value == Channel.VOID ) { @@ -339,7 +412,7 @@ class DataflowHelper { * when terminates bind the result value */ void afterStop(final DataflowProcessor processor) { - result.bind(accum) + Op.bind(result, accum) } boolean onException(final DataflowProcessor processor, final Throwable e) { @@ -349,7 +422,12 @@ class DataflowHelper { } } - chainImpl(channel, CH.create(), [listeners: [listener]], {true}) + final params = new OpParams() + .withInput(channel) + .withOutput(CH.create()) + .withListener(listener) + .withAccumulator(true) + chainImpl(params, {true}) } @PackageScope diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/GroupTupleOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/GroupTupleOp.groovy index 660ae6cf63..ddde2b825b 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/GroupTupleOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/GroupTupleOp.groovy @@ -224,7 +224,7 @@ class GroupTupleOp { target = CH.create() /* - * apply the logic the the source channel + * apply the logic to the source channel */ DataflowHelper.subscribeImpl(channel, [onNext: this.&collect, onComplete: this.&finalise]) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/MapOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/MapOp.groovy index 78e05a0777..ecea4fbc2f 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/MapOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/MapOp.groovy @@ -16,7 +16,6 @@ package nextflow.extension - import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowWriteChannel import groovyx.gpars.dataflow.expression.DataflowExpression @@ -53,16 +52,16 @@ class MapOp { final stopOnFirst = source instanceof DataflowExpression DataflowHelper.newOperator(source, target) { it -> - def result = mapper.call(it) - def proc = (DataflowProcessor) getDelegate() + final result = mapper.call(it) + final proc = (DataflowProcessor) getDelegate() // bind the result value if (result != Channel.VOID) - proc.bindOutput(result) + Op.bind(target, result) // when the `map` operator is applied to a dataflow flow variable // terminate the processor after the first emission -- Issue #44 - if( result == Channel.STOP || stopOnFirst ) + if( stopOnFirst ) proc.terminate() } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/Op.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/Op.groovy new file mode 100644 index 0000000000..7e63ea3f0b --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/extension/Op.groovy @@ -0,0 +1,166 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.extension + +import groovy.transform.CompileDynamic +import groovy.transform.CompileStatic +import groovy.transform.PackageScope +import groovy.util.logging.Slf4j +import groovyx.gpars.dataflow.DataflowWriteChannel +import groovyx.gpars.dataflow.operator.PoisonPill +import nextflow.Global +import nextflow.Session +import nextflow.prov.OperatorRun +import nextflow.prov.Prov +import nextflow.prov.Tracker + +/** + * Operator helpers methods + * + * @author Paolo Di Tommaso + */ +@Slf4j +@CompileStatic +class Op { + + static final @PackageScope ThreadLocal currentOperator = new ThreadLocal<>() + + static List unwrap(List messages) { + return messages.collect(it -> it instanceof Tracker.Msg ? it.value : it) + } + + static Object unwrap(Object it) { + return it instanceof Tracker.Msg ? it.value : it + } + + static Tracker.Msg wrap(Object obj) { + obj instanceof Tracker.Msg ? obj : Tracker.Msg.of(obj) + } + + static void bind(DataflowWriteChannel channel, Object msg) { + try { + if( msg instanceof PoisonPill ) + channel.bind(msg) + else + Prov.getTracker().bindOutput(currentOperator.get(), channel, msg) + } + catch (Throwable t) { + log.error("Unexpected resolving execution provenance: ${t.message}", t) + (Global.session as Session).abort(t) + } + } + + static Closure instrument(Closure op, boolean accumulator=false) { + return new InvokeOperatorAdapter(op, accumulator) + } + + static class InvokeOperatorAdapter extends Closure { + + private final Closure target + + private final boolean accumulator + + private OperatorRun previousRun + + private InvokeOperatorAdapter(Closure code, boolean accumulator) { + super(code.owner, code.thisObject) + this.target = code + this.target.delegate = code.delegate + this.target.setResolveStrategy(code.resolveStrategy) + this.accumulator = accumulator + } + + @Override + Class[] getParameterTypes() { + return target.getParameterTypes() + } + + @Override + int getMaximumNumberOfParameters() { + return target.getMaximumNumberOfParameters() + } + + @Override + Object getDelegate() { + return target.getDelegate() + } + + @Override + Object getProperty(String propertyName) { + return target.getProperty(propertyName) + } + + @Override + int getDirective() { + return target.getDirective() + } + + @Override + void setDelegate(Object delegate) { + target.setDelegate(delegate) + } + + @Override + void setDirective(int directive) { + target.setDirective(directive) + } + + @Override + void setResolveStrategy(int resolveStrategy) { + target.setResolveStrategy(resolveStrategy) + } + + @Override + void setProperty(String propertyName, Object newValue) { + target.setProperty(propertyName, newValue) + } + + @Override + @CompileDynamic + Object call(final Object... args) { + // when the accumulator flag true, re-use the previous run object + final run = !accumulator || previousRun==null + ? new OperatorRun() + : previousRun + // set as the current run in the thread local + currentOperator.set(run) + // map the inputs + final inputs = Prov.getTracker().receiveInputs(run, args.toList()) + final arr = inputs.toArray() + // todo: the spread operator should be replaced with proper array + final ret = target.call(*arr) + // track the previous run + if( accumulator ) + previousRun = run + // return the operation result + return ret + } + + Object call(Object args) { + // todo: this should invoke the above one + target.call(args) + } + + @Override + Object call() { + // todo: this should invoke the above one + target.call() + } + } + +} diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy index 3614de19db..28de8e1521 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy @@ -139,32 +139,31 @@ class OperatorImpl { newOperator(source, target, listener) { item -> - def result = closure != null ? closure.call(item) : item - def proc = ((DataflowProcessor) getDelegate()) + final result = closure != null ? closure.call(item) : item switch( result ) { case Collection: - result.each { it -> proc.bindOutput(it) } + result.each { it -> Op.bind(target,it) } break case (Object[]): - result.each { it -> proc.bindOutput(it) } + result.each { it -> Op.bind(target,it) } break case Map: - result.each { it -> proc.bindOutput(it) } + result.each { it -> Op.bind(target,it) } break case Map.Entry: - proc.bindOutput( (result as Map.Entry).key ) - proc.bindOutput( (result as Map.Entry).value ) + Op.bind(target, (result as Map.Entry).key ) + Op.bind(target, (result as Map.Entry).value ) break case Channel.VOID: break default: - proc.bindOutput(result) + Op.bind(target,result) } } diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskId.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskId.groovy index 3576b9fde7..bbb467ca5f 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskId.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskId.groovy @@ -19,6 +19,8 @@ package nextflow.processor import java.util.concurrent.atomic.AtomicInteger import groovy.transform.CompileStatic +import nextflow.util.TestOnly + /** * TaskRun unique identifier * @@ -32,12 +34,16 @@ class TaskId extends Number implements Comparable, Serializable, Cloneable { */ static final private AtomicInteger allCount = new AtomicInteger() + @TestOnly static void clear() { allCount.set(0) } + static TaskId next() { new TaskId(allCount.incrementAndGet()) } private final int value + int getValue() { value } + static TaskId of( value ) { if( value instanceof Integer ) return new TaskId(value) diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy index e0698be111..19ce31fd8d 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy @@ -15,9 +15,6 @@ */ package nextflow.processor -import nextflow.provenance.ProvTracker -import nextflow.trace.TraceRecord - import static nextflow.processor.ErrorStrategy.* import java.lang.reflect.InvocationTargetException @@ -36,6 +33,7 @@ import java.util.regex.Pattern import ch.artecat.grengine.Grengine import com.google.common.hash.HashCode import groovy.json.JsonOutput +import groovy.transform.Canonical import groovy.transform.CompileStatic import groovy.transform.Memoized import groovy.transform.PackageScope @@ -83,6 +81,7 @@ import nextflow.file.FilePatternSplitter import nextflow.file.FilePorter import nextflow.plugin.Plugins import nextflow.processor.tip.TaskTipProvider +import nextflow.prov.Prov import nextflow.script.BaseScript import nextflow.script.BodyDef import nextflow.script.ProcessConfig @@ -107,6 +106,7 @@ import nextflow.script.params.TupleInParam import nextflow.script.params.TupleOutParam import nextflow.script.params.ValueInParam import nextflow.script.params.ValueOutParam +import nextflow.trace.TraceRecord import nextflow.util.ArrayBag import nextflow.util.BlankSeparatedList import nextflow.util.CacheHelper @@ -134,6 +134,11 @@ class TaskProcessor { RunType(String str) { message=str }; } + @Canonical + static class FairEntry { + TaskRun task + Map emissions + } static final public String TASK_CONTEXT_PROPERTY_NAME = 'task' final private static Pattern ENV_VAR_NAME = ~/[a-zA-Z_]+[a-zA-Z0-9_]*/ @@ -144,6 +149,8 @@ class TaskProcessor { @TestOnly static TaskProcessor currentProcessor() { currentProcessor0 } + @TestOnly static Map allTasks = new HashMap<>() + /** * Keeps track of the task instance executed by the current thread */ @@ -252,16 +259,14 @@ class TaskProcessor { private static LockManager lockManager = new LockManager() - private List> fairBuffers = new ArrayList<>() + private List fairBuffers = new ArrayList<>() - private int currentEmission + private volatile int currentEmission private Boolean isFair0 private TaskArrayCollector arrayCollector - private ProvTracker provenance - private CompilerConfiguration compilerConfig() { final config = new CompilerConfiguration() config.addCompilationCustomizers( new ASTTransformationCustomizer(TaskTemplateVarsXform) ) @@ -321,7 +326,6 @@ class TaskProcessor { final arraySize = config.getArray() this.arrayCollector = arraySize > 0 ? new TaskArrayCollector(this, executor, arraySize) : null - this.provenance = session.getProvenance() } /** @@ -635,8 +639,9 @@ class TaskProcessor { final task = createTaskRun(params) // -- set the task instance as the current in this thread currentTask.set(task) + allTasks.put(task.id, task) // track the task provenance for the given inputs - final values = provenance.beforeRun(task, inputs) + final values = Prov.tracker.receiveInputs(task, inputs) // -- validate input lengths validateInputTuples(values) @@ -1479,19 +1484,20 @@ class TaskProcessor { synchronized (isFair0) { // decrement -1 because tasks are 1-based final index = task.index-1 + FairEntry entry = new FairEntry(task,emissions) // store the task emission values in a buffer - fairBuffers[index-currentEmission] = emissions + fairBuffers[index-currentEmission] = entry // check if the current task index matches the expected next emission index if( currentEmission == index ) { - while( emissions!=null ) { + while( entry!=null ) { // bind the emission values - bindOutputs0(emissions) + bindOutputs0(entry.emissions, entry.task) // remove the head and try with the following fairBuffers.remove(0) // increase the index of the next emission currentEmission++ // take the next emissions - emissions = fairBuffers[0] + entry = fairBuffers[0] } } } @@ -1524,7 +1530,7 @@ class TaskProcessor { // and result in a potential error. See https://github.com/nextflow-io/nextflow/issues/3768 final copy = x instanceof List && x instanceof Cloneable ? x.clone() : x // emit the final value - provenance.bindOutput(task, ch, copy) + Prov.tracker.bindOutput(task, ch, copy) } } @@ -2374,7 +2380,7 @@ class TaskProcessor { * @param task The {@code TaskRun} instance to finalize */ @PackageScope - final finalizeTask( TaskHandler handler) { + final finalizeTask(TaskHandler handler) { def task = handler.task log.trace "finalizing process > ${safeTaskName(task)} -- $task" diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy index 49851e035c..922d67a13a 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy @@ -38,6 +38,7 @@ import nextflow.exception.ProcessTemplateException import nextflow.exception.ProcessUnrecoverableException import nextflow.file.FileHelper import nextflow.file.FileHolder +import nextflow.prov.TrailRun import nextflow.script.BodyDef import nextflow.script.ScriptType import nextflow.script.TaskClosure @@ -59,7 +60,7 @@ import nextflow.spack.SpackCache */ @Slf4j -class TaskRun implements Cloneable { +class TaskRun implements Cloneable, TrailRun { final private ConcurrentHashMap cache0 = new ConcurrentHashMap() @@ -578,8 +579,8 @@ class TaskRun implements Cloneable { static final public String CMD_ENV = '.command.env' - String toString( ) { - "id: $id; name: $name; type: $type; exit: ${exitStatus==Integer.MAX_VALUE ? '-' : exitStatus}; error: $error; workDir: $workDir" + String toString() { + "TaskRun[id: $id; name: $name; type: $type; upstreams: ${upstreamTasks} exit: ${exitStatus==Integer.MAX_VALUE ? '-' : exitStatus}; error: $error; workDir: $workDir]" } diff --git a/modules/nextflow/src/main/groovy/nextflow/prov/OperatorRun.groovy b/modules/nextflow/src/main/groovy/nextflow/prov/OperatorRun.groovy new file mode 100644 index 0000000000..fbbb52101e --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/prov/OperatorRun.groovy @@ -0,0 +1,40 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.prov + +import groovy.transform.Canonical +import groovy.transform.CompileStatic + +/** + * Model an operator run + * + * @author Paolo Di Tommaso + */ +@Canonical +@CompileStatic +class OperatorRun implements TrailRun { + /** + * The list of (object) ids that was received as input by a operator run + */ + List inputIds = new ArrayList<>(10) + + @Override + String toString() { + "OperatorRun[id=${System.identityHashCode(this)}; inputs=${inputIds}]" + } +} diff --git a/modules/nextflow/src/main/groovy/nextflow/prov/Prov.groovy b/modules/nextflow/src/main/groovy/nextflow/prov/Prov.groovy new file mode 100644 index 0000000000..ddf0a02f88 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/prov/Prov.groovy @@ -0,0 +1,46 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.prov + + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.util.TestOnly +/** + * Provenance tracker facade class + * + * @author Paolo Di Tommaso + */ +@Slf4j +@CompileStatic +class Prov { + + static private volatile Tracker tracker0 + + static Tracker getTracker() { + if( tracker0==null ) + tracker0 = new Tracker() + return tracker0 + } + + @TestOnly + static void clear() { + tracker0 = null + } + +} diff --git a/modules/nextflow/src/main/groovy/nextflow/prov/Tracker.groovy b/modules/nextflow/src/main/groovy/nextflow/prov/Tracker.groovy new file mode 100644 index 0000000000..38d0d7d3db --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/prov/Tracker.groovy @@ -0,0 +1,162 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.prov + +import java.util.concurrent.ConcurrentHashMap + +import groovy.transform.Canonical +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import groovyx.gpars.dataflow.DataflowWriteChannel +import nextflow.extension.Op +import nextflow.processor.TaskId +import nextflow.processor.TaskRun +/** + * + * @author Paolo Di Tommaso + */ +@Slf4j +@CompileStatic +class Tracker { + + static @Canonical class Msg { + final Object value + + String toString() { + "Msg[id=${System.identityHashCode(this)}; value=${value}]" + } + + static Msg of(Object o) { + new Msg(o) + } + } + + /** + * Associate an output value with the corresponding task run that emitted it + */ + private Map messages = new ConcurrentHashMap<>() + + List receiveInputs(TaskRun task, List inputs) { + // find the upstream tasks id + findUpstreamTasks(task, inputs) + // log for debugging purposes + logInputs(task, inputs) + // the second entry of messages list represent the run inputs list + // apply the de-normalization before returning it + return Op.unwrap(inputs) + } + + private logInputs(TaskRun task, List inputs) { + if( log.isDebugEnabled() ) { + def msg = "Task input" + msg += "\n - id : ${task.id} " + msg += "\n - name : '${task.name}'" + msg += "\n - upstream: ${task.upstreamTasks*.value.join(',')}" + for( Object it : inputs ) { + msg += "\n<= ${it}" + } + log.debug(msg) + } + } + + private logInputs(OperatorRun run, List inputs) { + if( log.isDebugEnabled() ) { + def msg = "Operator input" + msg += "\n - id: ${System.identityHashCode(run)} " + for( Object it : inputs ) { + msg += "\n<= ${it}" + } + log.debug(msg) + } + } + + List receiveInputs(OperatorRun run, List inputs) { + // find the upstream tasks id + run.inputIds.addAll(inputs.collect(msg-> System.identityHashCode(msg))) + // log for debugging purposes + logInputs(run, inputs) + // the second entry of messages list represent the task inputs list + // apply the de-normalization before returning it + return Op.unwrap(inputs) + } + + protected void findUpstreamTasks(TaskRun task, List messages) { + // find upstream tasks and restore nulls + final result = new HashSet() + for( Object msg : messages ) { + if( msg==null ) + throw new IllegalArgumentException("Message cannot be a null object") + if( msg !instanceof Msg ) + continue + final msgId = System.identityHashCode(msg) + result.addAll(findUpstreamTasks0(msgId,result)) + } + // finally bind the result to the task record + task.upstreamTasks = result + } + + protected Set findUpstreamTasks0(final int msgId, Set upstream) { + final run = messages.get(msgId) + if( run instanceof TaskRun ) { + upstream.add(run.id) + return upstream + } + if( run instanceof OperatorRun ) { + for( Integer it : run.inputIds ) { + if( it!=msgId ) { + findUpstreamTasks0(it, upstream) + } + else { + log.debug "Skip duplicate provenance message id=${msgId}" + } + } + } + return upstream + } + + Msg bindOutput(TrailRun run, DataflowWriteChannel channel, Object out) { + assert run!=null, "Argument 'run' cannot be null" + assert channel!=null, "Argument 'channel' cannot be null" + + final msg = Op.wrap(out) + logOutput(run, msg) + // map the message with the run where it has been output + messages.put(System.identityHashCode(msg), run) + // now emit the value + channel.bind(msg) + return msg + } + + private void logOutput(TrailRun run, Msg msg) { + String str + if( run instanceof OperatorRun ) { + str = "Operator output" + str += "\n - id : ${System.identityHashCode(run)}" + } + else if( run instanceof TaskRun ) { + str = "Task output" + str += "\n - id : ${run.id}" + str += "\n - name: '${run.name}'" + } + else + throw new IllegalArgumentException("Unknown run type: ${run}") + str += "\n=> ${msg}" + log.debug(str) + } + +} diff --git a/modules/nextflow/src/main/groovy/nextflow/prov/TrailRun.groovy b/modules/nextflow/src/main/groovy/nextflow/prov/TrailRun.groovy new file mode 100644 index 0000000000..0f04c9e6d9 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/prov/TrailRun.groovy @@ -0,0 +1,25 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.prov + +/** + * + * @author Paolo Di Tommaso + */ +interface TrailRun { +} diff --git a/modules/nextflow/src/main/groovy/nextflow/provenance/ProvTracker.groovy b/modules/nextflow/src/main/groovy/nextflow/provenance/ProvTracker.groovy deleted file mode 100644 index 5eebc2edb1..0000000000 --- a/modules/nextflow/src/main/groovy/nextflow/provenance/ProvTracker.groovy +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright 2013-2024, Seqera Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package nextflow.provenance - -import java.util.concurrent.ConcurrentHashMap - -import groovy.transform.CompileStatic -import groovyx.gpars.dataflow.DataflowWriteChannel -import nextflow.processor.TaskId -import nextflow.processor.TaskRun -/** - * - * @author Paolo Di Tommaso - */ -@CompileStatic -class ProvTracker { - - static class NullMessage { } - - private Map messages = new ConcurrentHashMap<>() - - List beforeRun(TaskRun task, List messages) { - // find the upstream tasks id - findUpstreamTasks(task, messages) - // the second entry of messages list represent the task inputs list - // apply the de-normalization before returning it - return denormalizeMessages(messages) - } - - protected void findUpstreamTasks(TaskRun task, List messages) { - // find upstream tasks and restore nulls - final result = new HashSet() - for( Object msg : messages ) { - if( msg==null ) - throw new IllegalArgumentException("Message cannot be a null object") - final msgId = System.identityHashCode(msg) - result.addAll(findUpstreamTasks0(msgId,result)) - } - // finally bind the result to the task record - task.upstreamTasks = result - } - - protected Set findUpstreamTasks0(final int msgId, Set upstream) { - final task = messages.get(msgId) - if( task==null ) { - return upstream - } - if( task ) { - upstream.add(task.id) - return upstream - } - return upstream - } - - protected Object denormalizeMessage(Object msg) { - return msg !instanceof NullMessage ? msg : null - } - - protected List denormalizeMessages(List messages) { - return messages.collect(it-> denormalizeMessage(it)) - } - - protected Object normalizeMessage(Object message) { - // map a "null" value into an instance of "NullMessage" - // because it's needed the object identity to track the message flow - return message!=null ? message : new NullMessage() - } - - void bindOutput(TaskRun task, DataflowWriteChannel ch, Object msg) { - final value = normalizeMessage(msg) - // map the message with the run where it has been output - messages.put(System.identityHashCode(value), task) - // now emit the value - ch.bind(value) - } - -} diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/ConcatOpTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/ConcatOpTest.groovy index 5a40b18927..1600300d97 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/ConcatOpTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/ConcatOpTest.groovy @@ -25,7 +25,7 @@ import test.Dsl2Spec * @author Paolo Di Tommaso */ @Timeout(5) -class ConcatOp2Test extends Dsl2Spec { +class ConcatOpTest extends Dsl2Spec { def 'should concat two channel'() { diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/OpTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/OpTest.groovy new file mode 100644 index 0000000000..e55decaa7d --- /dev/null +++ b/modules/nextflow/src/test/groovy/nextflow/extension/OpTest.groovy @@ -0,0 +1,47 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.extension + + +import nextflow.prov.Prov +import spock.lang.Specification + +/** + * + * @author Paolo Di Tommaso + */ +class OpTest extends Specification { + + def 'should instrument a closure'() { + given: + def code = { int x, int y -> x+y } + def v1 = 1 + def v2 = 2 + + when: + def c = Op.instrument(code) + def z = c.call([v1, v2] as Object[]) + then: + z == 3 + and: + Op.currentOperator.get().inputIds == [ System.identityHashCode(v1), System.identityHashCode(v2) ] + + cleanup: + Prov.clear() + } +} diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/UntilManyOpTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/UntilManyOpTest.groovy index c64bee82ce..706771b1b9 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/UntilManyOpTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/UntilManyOpTest.groovy @@ -19,10 +19,13 @@ package nextflow.extension import nextflow.Channel import spock.lang.Specification +import spock.lang.Timeout + /** * * @author Paolo Di Tommaso */ +@Timeout(10) class UntilManyOpTest extends Specification { def 'should emit channel items until the condition is verified' () { @@ -94,5 +97,4 @@ class UntilManyOpTest extends Specification { Z.val == Channel.STOP } - } diff --git a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy new file mode 100644 index 0000000000..e4ee22054f --- /dev/null +++ b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy @@ -0,0 +1,206 @@ +package nextflow.prov + +import static test.TestHelper.* + +import nextflow.config.ConfigParser +import nextflow.processor.TaskId +import nextflow.processor.TaskProcessor +import test.Dsl2Spec +/** + * + * @author Paolo Di Tommaso + */ +class ProvTest extends Dsl2Spec { + + def setup() { + Prov.clear() + TaskId.clear() + TaskProcessor.allTasks.clear() + } + + ConfigObject globalConfig() { + new ConfigParser().parse(''' + process.fair = true + ''') + } + + def 'should chain two process'() { + + when: + dsl_eval(globalConfig(), ''' + workflow { + p1 | map { x-> x } | map { x-> x+1 } | p2 + } + + process p1 { + output: val(x) + exec: + x =1 + } + + process p2 { + input: val(x) + exec: + println x + } + ''') + + then: + def upstream = upstreamTasksOf('p2') + upstream.size() == 1 + upstream.first.name == 'p1' + } + + def 'should branch two process'() { + + when: + dsl_eval(globalConfig(), ''' + workflow { + channel.of(1,10,20) \ + | p1 \ + | branch { left: it <=10; right: it >10 } \ + | set { result } + + result.left | p2 + result.right | p3 + } + + process p1 { + input: val(x) + output: val(y) + exec: + y = x+1 + } + + process p2 { + input: val(x) + exec: + println x + } + + process p3 { + input: val(x) + exec: + println x + } + ''') + then: + def t1 = upstreamTasksOf('p2 (1)') + t1.first.name == 'p1 (1)' + t1.size() == 1 + + and: + def t2 = upstreamTasksOf('p3 (1)') + t2.first.name == 'p1 (2)' + t2.size() == 1 + + and: + def t3 = upstreamTasksOf('p3 (2)') + t3.first.name == 'p1 (3)' + t3.size() == 1 + } + + def 'should track provenance with flatMap operator' () { + when: + dsl_eval(globalConfig(), ''' + workflow { + channel.of(1,2) \ + | p1 \ + | flatMap \ + | p2 + } + + process p1 { + input: val(x) + output: val(y) + exec: + y = [x, x*x] + } + + process p2 { + input: val(x) + exec: + println x + } + ''') + then: + def t1 = upstreamTasksOf('p2 (1)') + t1.first.name == 'p1 (1)' + t1.size() == 1 + + and: + def t2 = upstreamTasksOf('p2 (2)') + t2.first.name == 'p1 (1)' + t2.size() == 1 + + and: + def t3 = upstreamTasksOf('p2 (3)') + t3.first.name == 'p1 (2)' + t3.size() == 1 + + and: + def t4 = upstreamTasksOf('p2 (4)') + t4.first.name == 'p1 (2)' + t4.size() == 1 + } + + def 'should track the provenance of two processes and reduce operator'() { + + when: + dsl_eval(globalConfig(), ''' + workflow { + channel.of(1,2,3) \ + | p1 \ + | reduce {a,b -> return a+b} \ + | p2 + } + + process p1 { + input: val(x) + output: val(y) + exec: + y = x + } + + process p2 { + input: val(x) + exec: + println x + } + ''') + + then: + def t1 = upstreamTasksOf('p2') + t1.name == ['p1 (1)', 'p1 (2)', 'p1 (3)'] + } + + def 'should track the provenance of two tasks and collectFile operator' () { + when: + dsl_eval(globalConfig(), ''' + workflow { + channel.of('a','b','c') \ + | p1 \ + | collectFile(name: 'sample.txt') \ + | p2 + } + + process p1 { + input: val(x) + output: val(y) + exec: + y = x + } + + process p2 { + input: file(x) + exec: + println x + } + ''') + + then: + def t1 = upstreamTasksOf('p2 (1)') + t1.name == ['p1 (1)', 'p1 (2)', 'p1 (3)'] + + } +} diff --git a/modules/nextflow/src/test/groovy/nextflow/prov/TrackerTest.groovy b/modules/nextflow/src/test/groovy/nextflow/prov/TrackerTest.groovy new file mode 100644 index 0000000000..8a2adc1518 --- /dev/null +++ b/modules/nextflow/src/test/groovy/nextflow/prov/TrackerTest.groovy @@ -0,0 +1,123 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.prov + +import groovyx.gpars.dataflow.DataflowQueue +import groovyx.gpars.dataflow.DataflowWriteChannel +import nextflow.processor.TaskConfig +import nextflow.processor.TaskId +import nextflow.processor.TaskProcessor +import nextflow.processor.TaskRun +import spock.lang.Specification + +/** + * + * @author Paolo Di Tommaso + */ +class TrackerTest extends Specification { + + def 'should normalize null values' () { + given: + def prov = new Tracker() + and: + def t1 = new TaskRun(id: new TaskId(1), processor: Mock(TaskProcessor), config: Mock(TaskConfig)) + def t2 = new TaskRun(id: new TaskId(2), processor: Mock(TaskProcessor), config: Mock(TaskConfig)) + and: + def msg1 = [Tracker.Msg.of('foo')] + def msg2 = [Tracker.Msg.of('foo'), Tracker.Msg.of(null)] + + when: + def result1 = prov.receiveInputs(t1, msg1) + then: + result1 == msg1.value + and: + t1.upstreamTasks == [] as Set + + when: + def result2 = prov.receiveInputs(t2, msg2) + then: + result2 == ['foo', null] + and: + t2.upstreamTasks == [] as Set + } + + def 'should bind value to task run' () { + given: + def prov = new Tracker() + and: + def t1 = new TaskRun(id: new TaskId(1), processor: Mock(TaskProcessor), config: Mock(TaskConfig)) + def c1 = new DataflowQueue() + def v1 = 'foo' + + when: + def m1 = prov.bindOutput(t1, c1, v1) + then: + c1.val.is(m1) + and: + prov.@messages.get(System.identityHashCode(m1)) == t1 + } + + def 'should determine upstream tasks' () { + given: + def prov = new Tracker() + and: + def t1 = new TaskRun(id: new TaskId(1), processor: Mock(TaskProcessor), config: Mock(TaskConfig)) + def t2 = new TaskRun(id: new TaskId(2), processor: Mock(TaskProcessor), config: Mock(TaskConfig)) + def t3 = new TaskRun(id: new TaskId(3), processor: Mock(TaskProcessor), config: Mock(TaskConfig)) + and: + def v1 = new Object() + def v2 = new Object() + + when: + def m1 = prov.bindOutput(t1, Mock(DataflowWriteChannel), v1) + def m2 = prov.bindOutput(t2, Mock(DataflowWriteChannel), v2) + and: + prov.receiveInputs(t3, [m1, m2]) + then: + t3.upstreamTasks == [t1.id, t2.id] as Set + } + + def 'should determine upstream task with operator' () { + given: + def prov = new Tracker() + and: + def v1 = Integer.valueOf(1) + def v2 = Integer.valueOf(2) + def v3 = Integer.valueOf(3) + and: + def t1 = new TaskRun(id: new TaskId(1), processor: Mock(TaskProcessor), config: Mock(TaskConfig)) + def p2 = new OperatorRun() + def t3 = new TaskRun(id: new TaskId(3), processor: Mock(TaskProcessor), config: Mock(TaskConfig)) + + when: + prov.receiveInputs(t1, []) + def m1 = prov.bindOutput(t1, Mock(DataflowWriteChannel), v1) + and: + prov.receiveInputs(p2, [m1]) + and: + def m2 = prov.bindOutput(p2, Mock(DataflowWriteChannel), v2) + and: + prov.receiveInputs(t3, [m2]) + and: + def m3 = prov.bindOutput(t3, Mock(DataflowWriteChannel), v3) + + then: + t3.upstreamTasks == [t1.id] as Set + } + +} diff --git a/modules/nextflow/src/test/groovy/nextflow/provenance/ProvTrackerTest.groovy b/modules/nextflow/src/test/groovy/nextflow/provenance/ProvTrackerTest.groovy deleted file mode 100644 index 571e127b8c..0000000000 --- a/modules/nextflow/src/test/groovy/nextflow/provenance/ProvTrackerTest.groovy +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright 2013-2024, Seqera Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package nextflow.provenance - -import groovyx.gpars.dataflow.DataflowQueue -import groovyx.gpars.dataflow.DataflowWriteChannel -import nextflow.processor.TaskId -import nextflow.processor.TaskRun -import spock.lang.Specification - -/** - * - * @author Paolo Di Tommaso - */ -class ProvTrackerTest extends Specification { - - def 'should normalize null values' () { - given: - def prov = new ProvTracker() - and: - def t1 = new TaskRun(id: new TaskId(1)) - def t2 = new TaskRun(id: new TaskId(2)) - and: - def msg1 = ['foo'] - def msg2 = ['foo', new ProvTracker.NullMessage()] - - when: - def result1 = prov.beforeRun(t1, msg1) - then: - result1 == msg1 - and: - t1.upstreamTasks == [] as Set - - when: - def result2 = prov.beforeRun(t2, msg2) - then: - result2 == ['foo', null] - and: - t2.upstreamTasks == [] as Set - } - - def 'should bind value to task run' () { - given: - def prov = new ProvTracker() - and: - def t1 = new TaskRun(id: new TaskId(1)) - def c1 = new DataflowQueue() - def v1 = 'foo' - - when: - prov.bindOutput(t1, c1, v1) - then: - c1.val == 'foo' - and: - prov.@messages.get(System.identityHashCode(v1)) == t1 - } - - def 'should determine upstream tasks' () { - given: - def prov = new ProvTracker() - and: - def t1 = new TaskRun(id: new TaskId(1)) - def t2 = new TaskRun(id: new TaskId(2)) - def t3 = new TaskRun(id: new TaskId(3)) - and: - def v1 = new Object() - def v2 = new Object() - def v3 = new Object() - - when: - prov.bindOutput(t1, Mock(DataflowWriteChannel), v1) - prov.bindOutput(t2, Mock(DataflowWriteChannel), v2) - and: - prov.beforeRun(t3, [v1, v2]) - then: - t3.upstreamTasks == [t1.id, t2.id] as Set - } - -} diff --git a/modules/nextflow/src/testFixtures/groovy/test/Dsl2Spec.groovy b/modules/nextflow/src/testFixtures/groovy/test/Dsl2Spec.groovy index a30c57f916..63c736b365 100644 --- a/modules/nextflow/src/testFixtures/groovy/test/Dsl2Spec.groovy +++ b/modules/nextflow/src/testFixtures/groovy/test/Dsl2Spec.groovy @@ -46,11 +46,14 @@ class Dsl2Spec extends BaseSpec { new MockScriptRunner().setScript(str).execute() } + def dsl_eval(Map config, String str) { + new MockScriptRunner(config).setScript(str).execute() + } + def dsl_eval(Path path) { new MockScriptRunner().setScript(path).execute() } - def dsl_eval(String entry, String str) { new MockScriptRunner() .setScript(str).execute(null, entry) diff --git a/modules/nextflow/src/testFixtures/groovy/test/MockHelpers.groovy b/modules/nextflow/src/testFixtures/groovy/test/MockHelpers.groovy index d4f5065c0d..be772ff133 100644 --- a/modules/nextflow/src/testFixtures/groovy/test/MockHelpers.groovy +++ b/modules/nextflow/src/testFixtures/groovy/test/MockHelpers.groovy @@ -127,7 +127,7 @@ class MockExecutor extends Executor { @Override TaskHandler createTaskHandler(TaskRun task) { - return new MockTaskHandler(task) + return new MockTaskHandler(task) } } diff --git a/modules/nextflow/src/testFixtures/groovy/test/TestHelper.groovy b/modules/nextflow/src/testFixtures/groovy/test/TestHelper.groovy index dce6834090..8ca3fcb841 100644 --- a/modules/nextflow/src/testFixtures/groovy/test/TestHelper.groovy +++ b/modules/nextflow/src/testFixtures/groovy/test/TestHelper.groovy @@ -15,6 +15,7 @@ */ package test + import java.nio.file.Files import java.nio.file.Path import java.util.zip.GZIPInputStream @@ -22,6 +23,9 @@ import java.util.zip.GZIPInputStream import com.google.common.jimfs.Configuration import com.google.common.jimfs.Jimfs import groovy.transform.Memoized +import nextflow.processor.TaskId +import nextflow.processor.TaskProcessor +import nextflow.processor.TaskRun /** * * @author Paolo Di Tommaso @@ -91,4 +95,36 @@ class TestHelper { // Convert the decoded bytes into a string return new String(decodedBytes); } + + static List upstreamTasksOf(v) { + if( v instanceof TaskRun ) + return upstreamTasksOf(v as TaskRun) + + if( v instanceof CharSequence ) { + TaskRun t = getTaskByName(v.toString()) + if( t ) + return upstreamTasksOf(t) + else + throw new IllegalArgumentException("Cannot find any task with name: $v") + } + + TaskRun t = getTaskById(v) + if( !t ) + throw new IllegalArgumentException("Cannot find any task with id: $v") + return upstreamTasksOf(t) + } + + static List upstreamTasksOf(TaskRun t) { + final ids = t.upstreamTasks ?: Set.of() + return ids.collect(it -> getTaskById(it)) + } + + static TaskRun getTaskByName(String name) { + TaskProcessor.allTasks.values().find( it -> it.name==name ) + } + + static TaskRun getTaskById(id) { + TaskProcessor.allTasks.get(TaskId.of(id)) + } + } From ddeee05f0b56c4f0764cccde4650516f493aa814 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Mon, 6 Jan 2025 18:10:05 +0800 Subject: [PATCH 04/66] Add toList and toSortedList operators Signed-off-by: Paolo Di Tommaso --- .../nextflow/extension/DataflowHelper.groovy | 64 ++- .../nextflow/extension/OperatorImpl.groovy | 68 ++- .../groovy/nextflow/extension/ToListOp.groovy | 27 +- .../extension/DataflowHelperTest.groovy | 75 ++- .../extension/OperatorImplTest.groovy | 438 +++++++++--------- .../test/groovy/nextflow/prov/ProvTest.groovy | 55 +++ 6 files changed, 480 insertions(+), 247 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy index 4543a89710..b7423fe8a4 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy @@ -53,9 +53,12 @@ class DataflowHelper { OpParams() { } OpParams(Map params) { - this.inputs = params.inputs as List ?: List.of() - this.outputs = params.outputs as List ?: List.of() - this.listeners = params.listeners as List ?: List.of() + if( params.inputs ) + this.inputs = params.inputs as List + if( params.outputs ) + this.outputs = params.outputs as List + if( params.listeners ) + this.listeners = params.listeners as List } OpParams withInput(DataflowReadChannel channel) { @@ -108,6 +111,44 @@ class DataflowHelper { } } + static class ReduceParams { + DataflowReadChannel source + DataflowVariable target + Object seed + Closure action + Closure beforeBind + + static ReduceParams build() { new ReduceParams() } + + ReduceParams withSource(DataflowReadChannel channel) { + assert channel!=null + this.source = channel + return this + } + + ReduceParams withTarget(DataflowVariable output) { + assert output!=null + this.target = output + return this + } + + ReduceParams withSeed(Object seed) { + this.seed = seed + return this + } + + ReduceParams withAction(Closure action) { + this.action = action + return this + } + + ReduceParams withBeforeBind(Closure beforeBind) { + this.beforeBind = beforeBind + return this + } + + } + private static Session getSession() { Global.getSession() as Session } /** @@ -383,10 +424,14 @@ class DataflowHelper { * @param closure * @return */ - static DataflowProcessor reduceImpl(final DataflowReadChannel channel, final DataflowVariable result, def seed, final Closure closure) { + static DataflowProcessor reduceImpl(ReduceParams opts) { + assert opts + assert opts.source, "Reduce 'source' channel cannot be null" + assert opts.target, "Reduce 'target' channel cannot be null" + assert opts.action, "Reduce 'action' closure cannot be null" // the *accumulator* value - def accum = seed + def accum = opts.seed // intercepts operator events def listener = new DataflowEventAdapter() { @@ -395,7 +440,7 @@ class DataflowHelper { */ void afterRun(final DataflowProcessor processor, final List messages) { final item = Op.unwrap(messages).get(0) - final value = accum == null ? item : closure.call(accum, item) + final value = accum == null ? item : opts.action.call(accum, item) if( value == Channel.VOID ) { // do nothing @@ -412,7 +457,10 @@ class DataflowHelper { * when terminates bind the result value */ void afterStop(final DataflowProcessor processor) { - Op.bind(result, accum) + final result = opts.beforeBind + ? opts.beforeBind.call(accum) + : accum + Op.bind(opts.target, result) } boolean onException(final DataflowProcessor processor, final Throwable e) { @@ -423,7 +471,7 @@ class DataflowHelper { } final params = new OpParams() - .withInput(channel) + .withInput(opts.source) .withOutput(CH.create()) .withListener(listener) .withAccumulator(true) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy index 28de8e1521..2de12c46a6 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy @@ -188,7 +188,11 @@ class OperatorImpl { throw new IllegalArgumentException('Operator `reduce` cannot be applied to a value channel') final target = new DataflowVariable() - reduceImpl( source, target, null, closure ) + reduceImpl( ReduceParams + .build() + .withSource(source) + .withTarget(target) + .withAction(closure) ) return target } @@ -211,7 +215,12 @@ class OperatorImpl { throw new IllegalArgumentException('Operator `reduce` cannot be applied to a value channel') final target = new DataflowVariable() - reduceImpl( source, target, seed, closure ) + reduceImpl( ReduceParams + .build() + .withSource(source) + .withTarget(target) + .withSeed(seed) + .withAction(closure) ) return target } @@ -488,8 +497,7 @@ class OperatorImpl { * @return A list holding all the items send over the channel */ DataflowWriteChannel toList(final DataflowReadChannel source) { - final target = ToListOp.apply(source) - return target + return new ToListOp(source).apply() } /** @@ -499,8 +507,7 @@ class OperatorImpl { * @return A list holding all the items send over the channel */ DataflowWriteChannel toSortedList(final DataflowReadChannel source, Closure closure = null) { - final target = new ToListOp(source, closure ?: true).apply() - return target as DataflowVariable + return new ToListOp(source, closure ?: true).apply() } /** @@ -538,9 +545,16 @@ class OperatorImpl { } } else { - reduceImpl(source, target, 0) { current, item -> + final action = { current, item -> discriminator == null || discriminator.invoke(criteria, item) ? current+1 : current } + reduceImpl(ReduceParams + .build() + .withSource(source) + .withTarget(target) + .withSeed(0) + .withAction(action) + ) } return target @@ -554,7 +568,11 @@ class OperatorImpl { */ DataflowWriteChannel min(final DataflowReadChannel source) { final target = new DataflowVariable() - reduceImpl(source, target, null) { min, val -> val val comparator.call(item) < comparator.call(min) ? item : min } } @@ -579,7 +597,11 @@ class OperatorImpl { } final target = new DataflowVariable() - reduceImpl(source, target, null, action) + reduceImpl(ReduceParams + .build() + .withSource(source) + .withTarget(target) + .withAction(action)) return target } @@ -592,7 +614,12 @@ class OperatorImpl { */ DataflowWriteChannel min(final DataflowReadChannel source, Comparator comparator) { final target = new DataflowVariable() - reduceImpl(source, target, null) { a, b -> comparator.compare(a,b)<0 ? a : b } + reduceImpl(ReduceParams + .build() + .withSource(source) + .withTarget(target) + .withAction{ a, b -> comparator.compare(a,b)<0 ? a : b } + ) return target } @@ -604,7 +631,12 @@ class OperatorImpl { */ DataflowWriteChannel max(final DataflowReadChannel source) { final target = new DataflowVariable() - reduceImpl(source,target, null) { max, val -> val>max ? val : max } + reduceImpl(ReduceParams + .build() + .withSource(source) + .withTarget(target) + .withAction { max, val -> val>max ? val : max } + ) return target } @@ -632,7 +664,11 @@ class OperatorImpl { } final target = new DataflowVariable() - reduceImpl(source, target, null, action) + reduceImpl(ReduceParams + .build() + .withSource(source) + .withTarget(target) + .withAction(action) ) return target } @@ -645,7 +681,11 @@ class OperatorImpl { */ DataflowVariable max(final DataflowReadChannel source, Comparator comparator) { final target = new DataflowVariable() - reduceImpl(source, target, null) { a, b -> comparator.compare(a,b)>0 ? a : b } + reduceImpl(ReduceParams + .build() + .withSource(source) + .withTarget(target) + .withAction { a, b -> comparator.compare(a,b)>0 ? a : b } ) return target } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/ToListOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/ToListOp.groovy index 163415b3f2..0d2c18893c 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/ToListOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/ToListOp.groovy @@ -52,19 +52,26 @@ class ToListOp { final result = new ArrayList(1) Map events = [:] events.onNext = { result.add(it) } - events.onComplete = { target.bind(result) } - DataflowHelper.subscribeImpl(source, events ) + events.onComplete = { Op.bind(target, result) } + DataflowHelper.subscribeImpl(source, events) return target } - DataflowHelper.reduceImpl(source, target, []) { List list, item -> list << item } - if( ordering ) { - final sort = { List list -> ordering instanceof Closure ? list.sort((Closure) ordering) : list.sort() } - return (DataflowVariable)target.then(sort) - } - else { - return target - } + Closure beforeBind = ordering + ? { List list -> ordering instanceof Closure ? list.sort((Closure) ordering) : list.sort() } + : null + + final reduce = DataflowHelper + .ReduceParams + .build() + .withSource(source) + .withTarget(target) + .withSeed([]) + .withBeforeBind(beforeBind) + .withAction{ List list, item -> list << item } + + DataflowHelper.reduceImpl(reduce) + return target } @Deprecated diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/DataflowHelperTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/DataflowHelperTest.groovy index 44c934d288..cd95f87fea 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/DataflowHelperTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/DataflowHelperTest.groovy @@ -16,10 +16,12 @@ package nextflow.extension +import groovyx.gpars.dataflow.DataflowQueue +import groovyx.gpars.dataflow.DataflowVariable +import groovyx.gpars.dataflow.operator.DataflowEventListener import nextflow.Session import spock.lang.Specification import spock.lang.Unroll - /** * * @author Paolo Di Tommaso @@ -70,4 +72,75 @@ class DataflowHelperTest extends Specification { [0,1,4] | ['A','B','C','D','E','F'] | ['A','B','E'] | ['C','D','F'] [0] | 'A' | ['A'] | [] } + + def 'should validate reduce params' () { + given: + def source = new DataflowQueue() + def target = new DataflowVariable() + def action = {-> 1} + def beforeBind = {-> 2} + def params = new DataflowHelper.ReduceParams() + .withSource(source) + .withTarget(target) + .withSeed('xyz') + .withAction(action) + .withBeforeBind(beforeBind) + + expect: + params.source.is(source) + params.target.is(target) + params.seed.is('xyz') + params.action.is(action) + params.beforeBind.is(beforeBind) + } + + def 'should validate operator params' () { + when: + def p1 = new DataflowHelper.OpParams().toMap() + then: + p1.inputs == List.of() + p1.outputs == List.of() + p1.listeners == List.of() + + when: + def s1 = new DataflowQueue() + def t1 = new DataflowQueue() + def l1 = Mock(DataflowEventListener) + and: + def p2 = new DataflowHelper.OpParams() + .withInput(s1) + .withOutput(t1) + .withListener(l1) + .withAccumulator(true) + then: + p2.inputs == List.of(s1) + p2.outputs == List.of(t1) + p2.listeners == List.of(l1) + p2.accumulator + and: + p2.toMap().inputs == List.of(s1) + p2.toMap().outputs == List.of(t1) + p2.toMap().listeners == List.of(l1) + + when: + def s2 = new DataflowQueue() + def t2 = new DataflowQueue() + def l2 = Mock(DataflowEventListener) + and: + def p3 = new DataflowHelper.OpParams() + .withInputs([s1,s2]) + .withOutputs([t1,t2]) + .withListeners([l1,l2]) + .withAccumulator(false) + then: + p3.inputs == List.of(s1,s2) + p3.outputs == List.of(t1,t2) + p3.listeners == List.of(l1,l2) + !p3.accumulator + and: + p3.toMap().inputs == List.of(s1,s2) + p3.toMap().outputs == List.of(t1,t2) + p3.toMap().listeners == List.of(l1,l2) + + } } diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/OperatorImplTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/OperatorImplTest.groovy index 10a96ba95a..c67af804d4 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/OperatorImplTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/OperatorImplTest.groovy @@ -19,9 +19,11 @@ package nextflow.extension import java.nio.file.Paths import groovyx.gpars.dataflow.DataflowQueue +import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowVariable import nextflow.Channel import nextflow.Session +import nextflow.prov.Tracker import spock.lang.Specification import spock.lang.Timeout /** @@ -35,56 +37,65 @@ class OperatorImplTest extends Specification { new Session() } + private mval(Object obj) { + if( obj instanceof DataflowReadChannel ) { + def result = obj.val + return result instanceof Tracker.Msg ? result.value : result + } + else + return obj + } + def testFilter() { when: def c1 = Channel.of(1,2,3,4,5).filter { it > 3 } then: - c1.val == 4 - c1.val == 5 - c1.val == Channel.STOP + mval(c1) == 4 + mval(c1) == 5 + mval(c1) == Channel.STOP when: def c2 = Channel.of('hola','hello','cioa','miao').filter { it =~ /^h.*/ } then: - c2.val == 'hola' - c2.val == 'hello' - c2.val == Channel.STOP + mval(c2) == 'hola' + mval(c2) == 'hello' + mval(c2) == Channel.STOP when: def c3 = Channel.of('hola','hello','cioa','miao').filter { it ==~ /^h.*/ } then: - c3.val == 'hola' - c3.val == 'hello' - c3.val == Channel.STOP + mval(c3) == 'hola' + mval(c3) == 'hello' + mval(c3) == Channel.STOP when: def c4 = Channel.of('hola','hello','cioa','miao').filter( ~/^h.*/ ) then: - c4.val == 'hola' - c4.val == 'hello' - c4.val == Channel.STOP + mval(c4) == 'hola' + mval(c4) == 'hello' + mval(c4) == Channel.STOP when: def c5 = Channel.of('hola',1,'cioa',2,3).filter( Number ) then: - c5.val == 1 - c5.val == 2 - c5.val == 3 - c5.val == Channel.STOP + mval(c5) == 1 + mval(c5) == 2 + mval(c5) == 3 + mval(c5) == Channel.STOP expect: - Channel.of(1,2,4,2,4,5,6,7,4).filter(1) .count().val == 1 - Channel.of(1,2,4,2,4,5,6,7,4).filter(2) .count().val == 2 - Channel.of(1,2,4,2,4,5,6,7,4).filter(4) .count().val == 3 + mval( Channel.of(1,2,4,2,4,5,6,7,4).filter(1) .count() ) == 1 + mval( Channel.of(1,2,4,2,4,5,6,7,4).filter(2) .count() ) == 2 + mval( Channel.of(1,2,4,2,4,5,6,7,4).filter(4) .count() ) == 3 } def testFilterWithValue() { expect: - Channel.value(3).filter { it>1 }.val == 3 - Channel.value(0).filter { it>1 }.val == Channel.STOP - Channel.value(Channel.STOP).filter { it>1 }.val == Channel.STOP + mval(Channel.value(3).filter { it>1 }) == 3 + mval(Channel.value(0).filter { it>1 }) == Channel.STOP + mval(Channel.value(Channel.STOP).filter { it>1 }) == Channel.STOP } def testSubscribe() { @@ -161,10 +172,10 @@ class OperatorImplTest extends Specification { when: def result = Channel.of(1,2,3).map { "Hello $it" } then: - result.val == 'Hello 1' - result.val == 'Hello 2' - result.val == 'Hello 3' - result.val == Channel.STOP + mval(result) == 'Hello 1' + mval(result) == 'Hello 2' + mval(result) == 'Hello 3' + mval(result) == Channel.STOP } def testMapWithVariable() { @@ -173,9 +184,9 @@ class OperatorImplTest extends Specification { when: def result = variable.map { it.reverse() } then: - result.val == 'olleH' - result.val == 'olleH' - result.val == 'olleH' + mval(result) == 'olleH' + mval(result) == 'olleH' + mval(result) == 'olleH' } def testMapParamExpanding () { @@ -183,10 +194,10 @@ class OperatorImplTest extends Specification { when: def result = Channel.of(1,2,3).map { [it, it] }.map { x, y -> x+y } then: - result.val == 2 - result.val == 4 - result.val == 6 - result.val == Channel.STOP + mval(result) == 2 + mval(result) == 4 + mval(result) == 6 + mval(result) == Channel.STOP } def testSkip() { @@ -194,9 +205,9 @@ class OperatorImplTest extends Specification { when: def result = Channel.of(1,2,3).map { it == 2 ? Channel.VOID : "Hello $it" } then: - result.val == 'Hello 1' - result.val == 'Hello 3' - result.val == Channel.STOP + mval(result) == 'Hello 1' + mval(result) == 'Hello 3' + mval(result) == Channel.STOP } @@ -206,13 +217,13 @@ class OperatorImplTest extends Specification { when: def result = Channel.of(1,2,3).flatMap { it -> [it, it*2] } then: - result.val == 1 - result.val == 2 - result.val == 2 - result.val == 4 - result.val == 3 - result.val == 6 - result.val == Channel.STOP + mval(result) == 1 + mval(result) == 2 + mval(result) == 2 + mval(result) == 4 + mval(result) == 3 + mval(result) == 6 + mval(result) == Channel.STOP } def testMapManyWithSingleton() { @@ -220,15 +231,15 @@ class OperatorImplTest extends Specification { when: def result = Channel.value([1,2,3]).flatMap() then: - result.val == 1 - result.val == 2 - result.val == 3 - result.val == Channel.STOP + mval(result) == 1 + mval(result) == 2 + mval(result) == 3 + mval(result) == Channel.STOP when: result = Channel.empty().flatMap() then: - result.val == Channel.STOP + mval(result) == Channel.STOP } @@ -237,11 +248,11 @@ class OperatorImplTest extends Specification { when: def result = Channel.of( [1,2], ['a','b'] ).flatMap { it -> [it, it.reverse()] } then: - result.val == [1,2] - result.val == [2,1] - result.val == ['a','b'] - result.val == ['b','a'] - result.val == Channel.STOP + mval(result) == [1, 2] + mval(result) == [2, 1] + mval(result) == ['a', 'b'] + mval(result) == ['b', 'a'] + mval(result) == Channel.STOP } def testMapManyDefault () { @@ -249,11 +260,11 @@ class OperatorImplTest extends Specification { when: def result = Channel.of( [1,2], ['a',['b','c']] ).flatMap() then: - result.val == 1 - result.val == 2 - result.val == 'a' - result.val == ['b','c'] // <-- nested list are preserved - result.val == Channel.STOP + mval(result) == 1 + mval(result) == 2 + mval(result) == 'a' + mval(result) == ['b', 'c'] // <-- nested list are preserved + mval(result) == Channel.STOP } def testMapManyWithHashArray () { @@ -261,13 +272,13 @@ class OperatorImplTest extends Specification { when: def result = Channel.of(1,2,3).flatMap { it -> [ k: it, v: it*2] } then: - result.val == new MapEntry('k',1) - result.val == new MapEntry('v',2) - result.val == new MapEntry('k',2) - result.val == new MapEntry('v',4) - result.val == new MapEntry('k',3) - result.val == new MapEntry('v',6) - result.val == Channel.STOP + mval(result) == new MapEntry('k',1) + mval(result) == new MapEntry('v',2) + mval(result) == new MapEntry('k',2) + mval(result) == new MapEntry('v',4) + mval(result) == new MapEntry('k',3) + mval(result) == new MapEntry('v',6) + mval(result) == Channel.STOP } @@ -280,33 +291,33 @@ class OperatorImplTest extends Specification { def result = channel.reduce { a, e -> a += e } channel << 1 << 2 << 3 << 4 << 5 << Channel.STOP then: - result.getVal() == 15 + mval(result) == 15 when: channel = Channel.of(1,2,3,4,5) result = channel.reduce { a, e -> a += e } then: - result.getVal() == 15 + mval(result) == 15 when: channel = Channel.create() result = channel.reduce { a, e -> a += e } channel << 99 << Channel.STOP then: - result.getVal() == 99 + mval(result) == 99 when: channel = Channel.create() result = channel.reduce { a, e -> a += e } channel << Channel.STOP then: - result.getVal() == null + mval(result) == null when: result = Channel.of(6,5,4,3,2,1).reduce { a, e -> Channel.STOP } then: - result.val == 6 + mval(result) == 6 } @@ -318,50 +329,50 @@ class OperatorImplTest extends Specification { def result = channel.reduce (1) { a, e -> a += e } channel << 1 << 2 << 3 << 4 << 5 << Channel.STOP then: - result.getVal() == 16 + mval(result) == 16 when: channel = Channel.create() result = channel.reduce (10) { a, e -> a += e } channel << Channel.STOP then: - result.getVal() == 10 + mval(result) == 10 when: result = Channel.of(6,5,4,3,2,1).reduce(0) { a, e -> a < 3 ? a+1 : Channel.STOP } then: - result.val == 3 + mval(result) == 3 } def testFirst() { expect: - Channel.of(3,6,4,5,4,3,4).first().val == 3 + mval(Channel.of(3,6,4,5,4,3,4).first()) == 3 } def testFirstWithCriteria() { expect: - Channel.of(3,6,4,5,4,3,4).first{ it>4 } .val == 6 + mval(Channel.of(3,6,4,5,4,3,4).first{ it>4 }) == 6 } def testFirstWithValue() { expect: - Channel.value(3).first().val == 3 - Channel.value(3).first{ it>1 }.val == 3 - Channel.value(3).first{ it>3 }.val == Channel.STOP - Channel.value(Channel.STOP).first { it>3 }.val == Channel.STOP + mval(Channel.value(3).first()) == 3 + mval(Channel.value(3).first{ it>1 }) == 3 + mval(Channel.value(3).first{ it>3 }) == Channel.STOP + mval(Channel.value(Channel.STOP).first { it>3 }) == Channel.STOP } def testFirstWithCondition() { expect: - Channel.of(3,6,4,5,4,3,4).first { it % 2 == 0 } .val == 6 - Channel.of( 'a', 'b', 'c', 1, 2 ).first( Number ) .val == 1 - Channel.of( 'a', 'b', 1, 2, 'aaa', 'bbb' ).first( ~/aa.*/ ) .val == 'aaa' - Channel.of( 'a', 'b', 1, 2, 'aaa', 'bbb' ).first( 1 ) .val == 1 + mval(Channel.of(3,6,4,5,4,3,4).first { it % 2 == 0 }) == 6 + mval(Channel.of( 'a', 'b', 'c', 1, 2 ).first( Number )) == 1 + mval(Channel.of( 'a', 'b', 1, 2, 'aaa', 'bbb' ).first( ~/aa.*/ )) == 'aaa' + mval(Channel.of( 'a', 'b', 1, 2, 'aaa', 'bbb' ).first( 1 )) == 1 } @@ -371,45 +382,45 @@ class OperatorImplTest extends Specification { when: def result = Channel.of(1,2,3,4,5,6).take(3) then: - result.val == 1 - result.val == 2 - result.val == 3 - result.val == Channel.STOP + mval(result) == 1 + mval(result) == 2 + mval(result) == 3 + mval(result) == Channel.STOP when: result = Channel.of(1).take(3) then: - result.val == 1 - result.val == Channel.STOP + mval(result) == 1 + mval(result) == Channel.STOP when: result = Channel.of(1,2,3).take(0) then: - result.val == Channel.STOP + mval(result) == Channel.STOP when: result = Channel.of(1,2,3).take(-1) then: - result.val == 1 - result.val == 2 - result.val == 3 - result.val == Channel.STOP + mval(result) == 1 + mval(result) == 2 + mval(result) == 3 + mval(result) == Channel.STOP when: result = Channel.of(1,2,3).take(3) then: - result.val == 1 - result.val == 2 - result.val == 3 - result.val == Channel.STOP + mval(result) == 1 + mval(result) == 2 + mval(result) == 3 + mval(result) == Channel.STOP } def testLast() { expect: - Channel.of(3,6,4,5,4,3,9).last().val == 9 - Channel.value('x').last().val == 'x' + mval(Channel.of(3,6,4,5,4,3,9).last()) == 9 + mval(Channel.value('x').last()) == 'x' } @@ -417,12 +428,12 @@ class OperatorImplTest extends Specification { def testCount() { expect: - Channel.of(4,1,7,5).count().val == 4 - Channel.of(4,1,7,1,1).count(1).val == 3 - Channel.of('a','c','c','q','b').count ( ~/c/ ) .val == 2 - Channel.value(5).count().val == 1 - Channel.value(5).count(5).val == 1 - Channel.value(5).count(6).val == 0 + mval(Channel.of(4,1,7,5).count()) == 4 + mval(Channel.of(4,1,7,1,1).count(1)) == 3 + mval(Channel.of('a','c','c','q','b').count ( ~/c/ )) == 2 + mval(Channel.value(5).count()) == 1 + mval(Channel.value(5).count(5)) == 1 + mval(Channel.value(5).count(6)) == 0 } def testToList() { @@ -430,23 +441,23 @@ class OperatorImplTest extends Specification { when: def channel = Channel.of(1,2,3) then: - channel.toList().val == [1,2,3] + mval(channel.toList()) == [1, 2, 3] when: channel = Channel.create() channel << Channel.STOP then: - channel.toList().val == [] + mval(channel.toList()) == [] when: channel = Channel.value(1) then: - channel.toList().val == [1] + mval(channel.toList()) == [1] when: channel = Channel.empty() then: - channel.toList().val == [] + mval(channel.toList()) == [] } def testToSortedList() { @@ -454,45 +465,44 @@ class OperatorImplTest extends Specification { when: def channel = Channel.of(3,1,4,2) then: - channel.toSortedList().val == [1,2,3,4] + mval(channel.toSortedList()) == [1, 2, 3, 4] when: - channel = Channel.create() - channel << Channel.STOP + channel = Channel.empty() then: - channel.toSortedList().val == [] + mval(channel.toSortedList()) == [] when: channel = Channel.of([1,'zeta'], [2,'gamma'], [3,'alpaha'], [4,'delta']) then: - channel.toSortedList { it[1] } .val == [[3,'alpaha'], [4,'delta'], [2,'gamma'], [1,'zeta'] ] + mval(channel.toSortedList { it[1] }) == [[3, 'alpaha'], [4, 'delta'], [2, 'gamma'], [1, 'zeta'] ] when: channel = Channel.value(1) then: - channel.toSortedList().val == [1] + mval(channel.toSortedList()) == [1] when: channel = Channel.empty() then: - channel.toSortedList().val == [] + mval(channel.toSortedList()) == [] } def testUnique() { expect: - Channel.of(1,1,1,5,7,7,7,3,3).unique().toList().val == [1,5,7,3] - Channel.of(1,3,4,5).unique { it%2 } .toList().val == [1,4] + mval(Channel.of(1,1,1,5,7,7,7,3,3).unique().toList()) == [1, 5, 7, 3] + mval(Channel.of(1,3,4,5).unique { it%2 } .toList()) == [1, 4] and: - Channel.of(1).unique().val == 1 - Channel.value(1).unique().val == 1 + mval(Channel.of(1).unique()) == 1 + mval(Channel.value(1).unique()) == 1 } def testDistinct() { expect: - Channel.of(1,1,2,2,2,3,1,1,2,2,3).distinct().toList().val == [1,2,3,1,2,3] - Channel.of(1,1,2,2,2,3,1,1,2,4,6).distinct { it%2 } .toList().val == [1,2,3,2] + mval(Channel.of(1,1,2,2,2,3,1,1,2,2,3).distinct().toList()) == [1, 2, 3, 1, 2, 3] + mval(Channel.of(1,1,2,2,2,3,1,1,2,4,6).distinct { it%2 } .toList()) == [1, 2, 3, 2] } @@ -501,54 +511,54 @@ class OperatorImplTest extends Specification { when: def r1 = Channel.of(1,2,3).flatten() then: - r1.val == 1 - r1.val == 2 - r1.val == 3 - r1.val == Channel.STOP + mval(r1) == 1 + mval(r1) == 2 + mval(r1) == 3 + mval(r1) == Channel.STOP when: def r2 = Channel.of([1,'a'], [2,'b']).flatten() then: - r2.val == 1 - r2.val == 'a' - r2.val == 2 - r2.val == 'b' - r2.val == Channel.STOP + mval(r2) == 1 + mval(r2) == 'a' + mval(r2) == 2 + mval(r2) == 'b' + mval(r2) == Channel.STOP when: def r3 = Channel.of( [1,2] as Integer[], [3,4] as Integer[] ).flatten() then: - r3.val == 1 - r3.val == 2 - r3.val == 3 - r3.val == 4 - r3.val == Channel.STOP + mval(r3) == 1 + mval(r3) == 2 + mval(r3) == 3 + mval(r3) == 4 + mval(r3) == Channel.STOP when: def r4 = Channel.of( [1,[2,3]], 4, [5,[6]] ).flatten() then: - r4.val == 1 - r4.val == 2 - r4.val == 3 - r4.val == 4 - r4.val == 5 - r4.val == 6 - r4.val == Channel.STOP + mval(r4) == 1 + mval(r4) == 2 + mval(r4) == 3 + mval(r4) == 4 + mval(r4) == 5 + mval(r4) == 6 + mval(r4) == Channel.STOP } def testFlattenWithSingleton() { when: def result = Channel.value([3,2,1]).flatten() then: - result.val == 3 - result.val == 2 - result.val == 1 - result.val == Channel.STOP + mval(result) == 3 + mval(result) == 2 + mval(result) == 1 + mval(result) == Channel.STOP when: result = Channel.empty().flatten() then: - result.val == Channel.STOP + mval(result) == Channel.STOP } def testCollate() { @@ -556,18 +566,18 @@ class OperatorImplTest extends Specification { when: def r1 = Channel.of(1,2,3,1,2,3,1).collate( 2, false ) then: - r1.val == [1,2] - r1.val == [3,1] - r1.val == [2,3] - r1.val == Channel.STOP + mval(r1) == [1, 2] + mval(r1) == [3, 1] + mval(r1) == [2, 3] + mval(r1) == Channel.STOP when: def r2 = Channel.of(1,2,3,1,2,3,1).collate( 3 ) then: - r2.val == [1,2,3] - r2.val == [1,2,3] - r2.val == [1] - r2.val == Channel.STOP + mval(r2) == [1, 2, 3] + mval(r2) == [1, 2, 3] + mval(r2) == [1] + mval(r2) == Channel.STOP } @@ -576,44 +586,44 @@ class OperatorImplTest extends Specification { when: def r1 = Channel.of(1,2,3,4).collate( 3, 1, false ) then: - r1.val == [1,2,3] - r1.val == [2,3,4] - r1.val == Channel.STOP + mval(r1) == [1, 2, 3] + mval(r1) == [2, 3, 4] + mval(r1) == Channel.STOP when: def r2 = Channel.of(1,2,3,4).collate( 3, 1, true ) then: - r2.val == [1,2,3] - r2.val == [2,3,4] - r2.val == [3,4] - r2.val == [4] - r2.val == Channel.STOP + mval(r2) == [1, 2, 3] + mval(r2) == [2, 3, 4] + mval(r2) == [3, 4] + mval(r2) == [4] + mval(r2) == Channel.STOP when: def r3 = Channel.of(1,2,3,4).collate( 3, 1 ) then: - r3.val == [1,2,3] - r3.val == [2,3,4] - r3.val == [3,4] - r3.val == [4] - r3.val == Channel.STOP + mval(r3) == [1, 2, 3] + mval(r3) == [2, 3, 4] + mval(r3) == [3, 4] + mval(r3) == [4] + mval(r3) == Channel.STOP when: def r4 = Channel.of(1,2,3,4).collate( 4,4 ) then: - r4.val == [1,2,3,4] - r4.val == Channel.STOP + mval(r4) == [1, 2, 3, 4] + mval(r4) == Channel.STOP when: def r5 = Channel.of(1,2,3,4).collate( 6,6 ) then: - r5.val == [1,2,3,4] - r5.val == Channel.STOP + mval(r5) == [1, 2, 3, 4] + mval(r5) == Channel.STOP when: def r6 = Channel.of(1,2,3,4).collate( 6,6,false ) then: - r6.val == Channel.STOP + mval(r6) == Channel.STOP } @@ -644,25 +654,25 @@ class OperatorImplTest extends Specification { when: def result = Channel.value(1).collate(1) then: - result.val == [1] - result.val == Channel.STOP + mval(result) == [1] + mval(result) == Channel.STOP when: result = Channel.value(1).collate(10) then: - result.val == [1] - result.val == Channel.STOP + mval(result) == [1] + mval(result) == Channel.STOP when: result = Channel.value(1).collate(10, true) then: - result.val == [1] - result.val == Channel.STOP + mval(result) == [1] + mval(result) == Channel.STOP when: result = Channel.value(1).collate(10, false) then: - result.val == Channel.STOP + mval(result) == Channel.STOP } def testMix() { @@ -687,7 +697,7 @@ class OperatorImplTest extends Specification { when: def result = Channel.value(1).mix( Channel.of(2,3) ) then: - result.toList().val.sort() == [1,2,3] + mval(result.toList().sort()) == [1, 2, 3] } @@ -741,12 +751,12 @@ class OperatorImplTest extends Specification { def result = ch1.cross(ch2) then: - result.val == [ [1, 'x'], [1,11] ] - result.val == [ [1, 'x'], [1,13] ] - result.val == [ [2, 'y'], [2,21] ] - result.val == [ [2, 'y'], [2,22] ] - result.val == [ [2, 'y'], [2,23] ] - result.val == Channel.STOP + mval(result) == [[1, 'x'], [1, 11] ] + mval(result) == [[1, 'x'], [1, 13] ] + mval(result) == [[2, 'y'], [2, 21] ] + mval(result) == [[2, 'y'], [2, 22] ] + mval(result) == [[2, 'y'], [2, 23] ] + mval(result) == Channel.STOP } @@ -761,9 +771,9 @@ class OperatorImplTest extends Specification { def result = ch1.cross(ch2) then: - result.val == [ ['PF00006', 'PF00006.sp_lib'], ['PF00006', 'PF00006_mafft.aln'] ] - result.val == [ ['PF00006', 'PF00006.sp_lib'], ['PF00006', 'PF00006_clustalo.aln'] ] - result.val == Channel.STOP + mval(result) == [['PF00006', 'PF00006.sp_lib'], ['PF00006', 'PF00006_mafft.aln'] ] + mval(result) == [['PF00006', 'PF00006.sp_lib'], ['PF00006', 'PF00006_clustalo.aln'] ] + mval(result) == Channel.STOP } @@ -779,9 +789,9 @@ class OperatorImplTest extends Specification { def result = ch1.cross(ch2) then: - result.val == [ ['PF00006', 'PF00006.sp_lib'], ['PF00006', 'PF00006_mafft.aln'] ] - result.val == [ ['PF00006', 'PF00006.sp_lib'], ['PF00006', 'PF00006_clustalo.aln'] ] - result.val == Channel.STOP + mval(result) == [['PF00006', 'PF00006.sp_lib'], ['PF00006', 'PF00006_mafft.aln'] ] + mval(result) == [['PF00006', 'PF00006.sp_lib'], ['PF00006', 'PF00006_clustalo.aln'] ] + mval(result) == Channel.STOP } @@ -793,13 +803,13 @@ class OperatorImplTest extends Specification { def c2 = Channel.of('a','b','c') def all = c1.concat(c2) then: - all.val == 1 - all.val == 2 - all.val == 3 - all.val == 'a' - all.val == 'b' - all.val == 'c' - all.val == Channel.STOP + mval(all) == 1 + mval(all) == 2 + mval(all) == 3 + mval(all) == 'a' + mval(all) == 'b' + mval(all) == 'c' + mval(all) == Channel.STOP when: def d1 = Channel.create() @@ -811,14 +821,14 @@ class OperatorImplTest extends Specification { Thread.start { sleep 100; d1 << 1 << 2 << Channel.STOP } then: - result.val == 1 - result.val == 2 - result.val == 'a' - result.val == 'b' - result.val == 'c' - result.val == 'p' - result.val == 'q' - result.val == Channel.STOP + mval(result) == 1 + mval(result) == 2 + mval(result) == 'a' + mval(result) == 'b' + mval(result) == 'c' + mval(result) == 'p' + mval(result) == 'q' + mval(result) == Channel.STOP } @@ -826,10 +836,10 @@ class OperatorImplTest extends Specification { when: def result = Channel.value(1).concat( Channel.of(2,3) ) then: - result.val == 1 - result.val == 2 - result.val == 3 - result.val == Channel.STOP + mval(result) == 1 + mval(result) == 2 + mval(result) == 3 + mval(result) == Channel.STOP } diff --git a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy index e4ee22054f..f13bd7a7b2 100644 --- a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy @@ -203,4 +203,59 @@ class ProvTest extends Dsl2Spec { t1.name == ['p1 (1)', 'p1 (2)', 'p1 (3)'] } + + def 'should track the provenance of two tasks and toList operator' () { + when: + dsl_eval(globalConfig(), ''' + workflow { + channel.of('a','b','c') | p1 | toList | p2 + } + + process p1 { + input: val(x) + output: val(y) + exec: + y = x + } + + process p2 { + input: val(x) + exec: + println x + } + ''') + + then: + def t1 = upstreamTasksOf('p2') + t1.name == ['p1 (1)', 'p1 (2)', 'p1 (3)'] + + } + + def 'should track the provenance of two tasks and toSortedList operator' () { + when: + dsl_eval(globalConfig(), ''' + workflow { + channel.of('a','b','c') | p1 | toList | p2 + } + + process p1 { + input: val(x) + output: val(y) + exec: + y = x + } + + process p2 { + input: val(x) + exec: + println x + } + ''') + + then: + def t1 = upstreamTasksOf('p2') + t1.name == ['p1 (1)', 'p1 (2)', 'p1 (3)'] + + } + } From e4e505a4978b48cd96ef8d1aafc8d85add31e6fd Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 7 Jan 2025 20:06:24 +0800 Subject: [PATCH 05/66] Refactor redice operator as its own class Signed-off-by: Paolo Di Tommaso --- .../nextflow/extension/DataflowHelper.groovy | 100 ------------ .../nextflow/extension/OperatorImpl.groovy | 76 +++------- .../groovy/nextflow/extension/ReduceOp.groovy | 143 ++++++++++++++++++ .../groovy/nextflow/extension/ToListOp.groovy | 10 +- .../extension/DataflowHelperTest.groovy | 22 --- .../extension/OperatorImplTest.groovy | 1 - 6 files changed, 171 insertions(+), 181 deletions(-) create mode 100644 modules/nextflow/src/main/groovy/nextflow/extension/ReduceOp.groovy diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy index b7423fe8a4..aad41407aa 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy @@ -111,44 +111,6 @@ class DataflowHelper { } } - static class ReduceParams { - DataflowReadChannel source - DataflowVariable target - Object seed - Closure action - Closure beforeBind - - static ReduceParams build() { new ReduceParams() } - - ReduceParams withSource(DataflowReadChannel channel) { - assert channel!=null - this.source = channel - return this - } - - ReduceParams withTarget(DataflowVariable output) { - assert output!=null - this.target = output - return this - } - - ReduceParams withSeed(Object seed) { - this.seed = seed - return this - } - - ReduceParams withAction(Closure action) { - this.action = action - return this - } - - ReduceParams withBeforeBind(Closure beforeBind) { - this.beforeBind = beforeBind - return this - } - - } - private static Session getSession() { Global.getSession() as Session } /** @@ -416,68 +378,6 @@ class DataflowHelper { newOperator(params, new ChainWithClosure(closure)) } - /** - * Implements the {@code #reduce} operator - * - * @param channel - * @param seed - * @param closure - * @return - */ - static DataflowProcessor reduceImpl(ReduceParams opts) { - assert opts - assert opts.source, "Reduce 'source' channel cannot be null" - assert opts.target, "Reduce 'target' channel cannot be null" - assert opts.action, "Reduce 'action' closure cannot be null" - - // the *accumulator* value - def accum = opts.seed - - // intercepts operator events - def listener = new DataflowEventAdapter() { - /* - * call the passed closure each time - */ - void afterRun(final DataflowProcessor processor, final List messages) { - final item = Op.unwrap(messages).get(0) - final value = accum == null ? item : opts.action.call(accum, item) - - if( value == Channel.VOID ) { - // do nothing - } - else if( value == Channel.STOP ) { - processor.terminate() - } - else { - accum = value - } - } - - /* - * when terminates bind the result value - */ - void afterStop(final DataflowProcessor processor) { - final result = opts.beforeBind - ? opts.beforeBind.call(accum) - : accum - Op.bind(opts.target, result) - } - - boolean onException(final DataflowProcessor processor, final Throwable e) { - log.error("@unknown", e) - session.abort(e) - return true; - } - } - - final params = new OpParams() - .withInput(opts.source) - .withOutput(CH.create()) - .withListener(listener) - .withAccumulator(true) - chainImpl(params, {true}) - } - @PackageScope @CompileStatic static KeyPair makeKey(List pivot, entry) { diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy index 2de12c46a6..4072acfcd4 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy @@ -187,13 +187,10 @@ class OperatorImpl { if( source instanceof DataflowExpression ) throw new IllegalArgumentException('Operator `reduce` cannot be applied to a value channel') - final target = new DataflowVariable() - reduceImpl( ReduceParams - .build() + ReduceOp .create() .withSource(source) - .withTarget(target) - .withAction(closure) ) - return target + .withAction(closure) + .apply() } /** @@ -214,14 +211,11 @@ class OperatorImpl { if( source instanceof DataflowExpression ) throw new IllegalArgumentException('Operator `reduce` cannot be applied to a value channel') - final target = new DataflowVariable() - reduceImpl( ReduceParams - .build() + ReduceOp .create() .withSource(source) - .withTarget(target) .withSeed(seed) - .withAction(closure) ) - return target + .withAction(closure) + .apply() } DataflowWriteChannel collectFile( final DataflowReadChannel source, final Closure closure = null ) { @@ -548,13 +542,13 @@ class OperatorImpl { final action = { current, item -> discriminator == null || discriminator.invoke(criteria, item) ? current+1 : current } - reduceImpl(ReduceParams - .build() + + ReduceOp .create() .withSource(source) .withTarget(target) .withSeed(0) .withAction(action) - ) + .apply() } return target @@ -567,13 +561,10 @@ class OperatorImpl { * @return A {@code DataflowVariable} returning the minimum value */ DataflowWriteChannel min(final DataflowReadChannel source) { - final target = new DataflowVariable() - reduceImpl( ReduceParams - .build() + ReduceOp .create() .withSource(source) - .withTarget(target) - .withAction{ min, val -> val val comparator.call(a,b) < 0 ? a : b } } - final target = new DataflowVariable() - reduceImpl(ReduceParams - .build() + ReduceOp .create() .withSource(source) - .withTarget(target) - .withAction(action)) - return target + .withAction(action) + .apply() } /** @@ -613,14 +601,10 @@ class OperatorImpl { * @return A {@code DataflowVariable} returning the minimum value */ DataflowWriteChannel min(final DataflowReadChannel source, Comparator comparator) { - final target = new DataflowVariable() - reduceImpl(ReduceParams - .build() + ReduceOp .create() .withSource(source) - .withTarget(target) .withAction{ a, b -> comparator.compare(a,b)<0 ? a : b } - ) - return target + .apply() } /** @@ -630,14 +614,10 @@ class OperatorImpl { * @return A {@code DataflowVariable} emitting the maximum value */ DataflowWriteChannel max(final DataflowReadChannel source) { - final target = new DataflowVariable() - reduceImpl(ReduceParams - .build() + ReduceOp .create() .withSource(source) - .withTarget(target) .withAction { max, val -> val>max ? val : max } - ) - return target + .apply() } /** @@ -663,13 +643,10 @@ class OperatorImpl { throw new IllegalArgumentException("Comparator closure can accept at most 2 arguments") } - final target = new DataflowVariable() - reduceImpl(ReduceParams - .build() + ReduceOp .create() .withSource(source) - .withTarget(target) - .withAction(action) ) - return target + .withAction(action) + .apply() } /** @@ -680,13 +657,10 @@ class OperatorImpl { * @return A {@code DataflowVariable} emitting the maximum value */ DataflowVariable max(final DataflowReadChannel source, Comparator comparator) { - final target = new DataflowVariable() - reduceImpl(ReduceParams - .build() + ReduceOp .create() .withSource(source) - .withTarget(target) - .withAction { a, b -> comparator.compare(a,b)>0 ? a : b } ) - return target + .withAction { a, b -> comparator.compare(a,b)>0 ? a : b } + .apply() } /** diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/ReduceOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/ReduceOp.groovy new file mode 100644 index 0000000000..00156d7d7c --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/extension/ReduceOp.groovy @@ -0,0 +1,143 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.extension + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import groovyx.gpars.dataflow.DataflowReadChannel +import groovyx.gpars.dataflow.DataflowVariable +import groovyx.gpars.dataflow.operator.DataflowEventAdapter +import groovyx.gpars.dataflow.operator.DataflowProcessor +import nextflow.Channel +import nextflow.Global +import nextflow.Session + +import static nextflow.extension.DataflowHelper.chainImpl + +/** + * Implements reduce operator logic + * + * @author Paolo Di Tommaso + */ +@Slf4j +@CompileStatic +@CompileStatic +class ReduceOp { + + private DataflowReadChannel source + private DataflowVariable target + private Object seed + private Closure action + private Closure beforeBind + + private ReduceOp() { } + + ReduceOp(DataflowReadChannel source) { + this.source = source + } + + static ReduceOp create() { + new ReduceOp() + } + + ReduceOp withSource(DataflowReadChannel source) { + this.source = source + return this + } + + ReduceOp withTarget(DataflowVariable target) { + this.target = target + return this + } + + ReduceOp withSeed(Object seed) { + this.seed = seed + return this + } + + ReduceOp withAction(Closure action) { + this.action = action + return this + } + + ReduceOp withBeforeBind(Closure beforeBind) { + this.beforeBind = beforeBind + return this + } + + private Session getSession() { + return Global.session as Session + } + + DataflowVariable apply() { + if( source==null ) + throw new IllegalArgumentException("Missing reduce operator source channel") + if( target==null ) + target = new DataflowVariable() + + // the *accumulator* value + def accum = this.seed + + // intercepts operator events + def listener = new DataflowEventAdapter() { + /* + * call the passed closure each time + */ + void afterRun(final DataflowProcessor processor, final List messages) { + final item = Op.unwrap(messages).get(0) + final value = accum == null ? item : action.call(accum, item) + + if( value == Channel.VOID ) { + // do nothing + } + else if( value == Channel.STOP ) { + processor.terminate() + } + else { + accum = value + } + } + + /* + * when terminates bind the result value + */ + void afterStop(final DataflowProcessor processor) { + final result = beforeBind + ? beforeBind.call(accum) + : accum + Op.bind(target, result) + } + + boolean onException(final DataflowProcessor processor, final Throwable e) { + log.error("@unknown", e) + session.abort(e) + return true; + } + } + + final params = new DataflowHelper.OpParams() + .withInput(source) + .withOutput(CH.create()) + .withListener(listener) + .withAccumulator(true) + chainImpl(params, {true}) + + return target + } + +} diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/ToListOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/ToListOp.groovy index 0d2c18893c..16bdb74ab1 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/ToListOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/ToListOp.groovy @@ -61,17 +61,13 @@ class ToListOp { ? { List list -> ordering instanceof Closure ? list.sort((Closure) ordering) : list.sort() } : null - final reduce = DataflowHelper - .ReduceParams - .build() + ReduceOp .create() .withSource(source) .withTarget(target) - .withSeed([]) + .withSeed(new ArrayList()) .withBeforeBind(beforeBind) .withAction{ List list, item -> list << item } - - DataflowHelper.reduceImpl(reduce) - return target + .apply() } @Deprecated diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/DataflowHelperTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/DataflowHelperTest.groovy index cd95f87fea..ff7edfae4e 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/DataflowHelperTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/DataflowHelperTest.groovy @@ -17,7 +17,6 @@ package nextflow.extension import groovyx.gpars.dataflow.DataflowQueue -import groovyx.gpars.dataflow.DataflowVariable import groovyx.gpars.dataflow.operator.DataflowEventListener import nextflow.Session import spock.lang.Specification @@ -73,27 +72,6 @@ class DataflowHelperTest extends Specification { [0] | 'A' | ['A'] | [] } - def 'should validate reduce params' () { - given: - def source = new DataflowQueue() - def target = new DataflowVariable() - def action = {-> 1} - def beforeBind = {-> 2} - def params = new DataflowHelper.ReduceParams() - .withSource(source) - .withTarget(target) - .withSeed('xyz') - .withAction(action) - .withBeforeBind(beforeBind) - - expect: - params.source.is(source) - params.target.is(target) - params.seed.is('xyz') - params.action.is(action) - params.beforeBind.is(beforeBind) - } - def 'should validate operator params' () { when: def p1 = new DataflowHelper.OpParams().toMap() diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/OperatorImplTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/OperatorImplTest.groovy index c67af804d4..0c581e8c19 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/OperatorImplTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/OperatorImplTest.groovy @@ -489,7 +489,6 @@ class OperatorImplTest extends Specification { } - def testUnique() { expect: mval(Channel.of(1,1,1,5,7,7,7,3,3).unique().toList()) == [1, 5, 7, 3] From e613123938b8e5ade98cd88bd435f2b968e10b18 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 7 Jan 2025 21:58:13 +0800 Subject: [PATCH 06/66] Fix tests Signed-off-by: Paolo Di Tommaso --- .../nextflow/extension/ChannelEx.groovy | 15 +- .../nextflow/extension/DataflowHelper.groovy | 8 +- .../main/groovy/nextflow/prov/Tracker.groovy | 12 +- .../nextflow/extension/BranchOpTest.groovy | 84 +- .../nextflow/extension/BufferOpTest.groovy | 72 +- .../extension/CollectFileOperatorTest.groovy | 59 +- .../nextflow/extension/CollectOpTest.groovy | 43 +- .../nextflow/extension/CombineOpTest.groovy | 52 +- .../nextflow/extension/ConcatOpTest.groovy | 34 +- .../extension/CountFastaOpTest.groovy | 7 +- .../DataflowMathExtensionTest.groovy | 63 +- .../DataflowMergeExtensionTest.groovy | 64 +- .../extension/DataflowTapExtensionTest.groovy | 46 +- .../nextflow/extension/JoinOpTest.groovy | 67 +- .../nextflow/extension/MixOpTest.groovy | 7 +- .../extension/OperatorImplTest.groovy | 742 ++++++++---------- .../extension/RandomSampleTest.groovy | 10 +- .../nextflow/extension/SetOpTest.groovy | 22 +- .../extension/SplitFastaOperatorTest.groovy | 46 +- .../extension/SplitFastqOperatorTest.groovy | 42 +- .../extension/ViewOperatorTest.groovy | 28 +- .../plugin/ChannelFactoryInstanceTest.groovy | 24 +- .../plugin/PluginExtensionProviderTest.groovy | 6 +- 23 files changed, 688 insertions(+), 865 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/ChannelEx.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/ChannelEx.groovy index ba68ef0390..121620bc10 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/ChannelEx.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/ChannelEx.groovy @@ -16,23 +16,23 @@ package nextflow.extension +import static nextflow.util.LoggerHelper.* + import groovy.transform.CompileDynamic import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import groovyx.gpars.agent.Agent +import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowWriteChannel -import nextflow.Channel -import nextflow.NF import nextflow.dag.NodeMarker import nextflow.exception.ScriptRuntimeException +import nextflow.prov.Tracker import nextflow.script.ChainableDef import nextflow.script.ChannelOut import nextflow.script.ComponentDef import nextflow.script.CompositeDef import nextflow.script.ExecutionStack import org.codehaus.groovy.runtime.InvokerHelper -import static nextflow.util.LoggerHelper.fmtType - /** * Implements dataflow channel extension methods * @@ -207,4 +207,11 @@ class ChannelEx { left.add(right) } + static Object unwrap(DataflowReadChannel self) { + final result = self.getVal() + return result instanceof Tracker.Msg + ? result.value + : result + } + } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy index aad41407aa..5b68f56c22 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy @@ -211,7 +211,7 @@ class DataflowHelper { static DataflowProcessor newOperator( Map params, Closure code ) { // -- add a default error listener - if( !params.containsKey('listeners') ) { + if( !params.listeners ) { // add the default error handler params.listeners = [ DEF_ERROR_LISTENER ] } @@ -367,9 +367,9 @@ class DataflowHelper { @Deprecated static DataflowProcessor chainImpl(final DataflowReadChannel source, final DataflowWriteChannel target, final Map params, final Closure closure) { - final Map parameters = new HashMap(params) - parameters.put("inputs", asList(source)) - parameters.put("outputs", asList(target)) + final OpParams parameters = new OpParams() + .withInput(source) + .withOutput(target) newOperator(parameters, new ChainWithClosure(closure)) } diff --git a/modules/nextflow/src/main/groovy/nextflow/prov/Tracker.groovy b/modules/nextflow/src/main/groovy/nextflow/prov/Tracker.groovy index 38d0d7d3db..8d9a6329a3 100644 --- a/modules/nextflow/src/main/groovy/nextflow/prov/Tracker.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/prov/Tracker.groovy @@ -62,7 +62,7 @@ class Tracker { } private logInputs(TaskRun task, List inputs) { - if( log.isDebugEnabled() ) { + if( log.isTraceEnabled() ) { def msg = "Task input" msg += "\n - id : ${task.id} " msg += "\n - name : '${task.name}'" @@ -70,18 +70,18 @@ class Tracker { for( Object it : inputs ) { msg += "\n<= ${it}" } - log.debug(msg) + log.trace(msg) } } private logInputs(OperatorRun run, List inputs) { - if( log.isDebugEnabled() ) { + if( log.isTraceEnabled() ) { def msg = "Operator input" msg += "\n - id: ${System.identityHashCode(run)} " for( Object it : inputs ) { msg += "\n<= ${it}" } - log.debug(msg) + log.trace(msg) } } @@ -122,7 +122,7 @@ class Tracker { findUpstreamTasks0(it, upstream) } else { - log.debug "Skip duplicate provenance message id=${msgId}" + log.trace "Skip duplicate provenance message id=${msgId}" } } } @@ -156,7 +156,7 @@ class Tracker { else throw new IllegalArgumentException("Unknown run type: ${run}") str += "\n=> ${msg}" - log.debug(str) + log.trace(str) } } diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/BranchOpTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/BranchOpTest.groovy index 13d3c9f49f..29269e8240 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/BranchOpTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/BranchOpTest.groovy @@ -48,14 +48,14 @@ class BranchOpTest extends Dsl2Spec { then: result.size() == 3 and: - result[0].val == 0 - result[0].val == Channel.STOP + result[0].unwrap() == 0 + result[0].unwrap() == Channel.STOP and: - result[1].val == 1 - result[1].val == Channel.STOP + result[1].unwrap() == 1 + result[1].unwrap() == Channel.STOP and: - result[2].val == 2 - result[2].val == Channel.STOP + result[2].unwrap() == 2 + result[2].unwrap() == Channel.STOP } def 'should branch and capture default' () { @@ -71,12 +71,12 @@ class BranchOpTest extends Dsl2Spec { then: result.size() == 2 and: - result[0].val == 10 - result[0].val == Channel.STOP + result[0].unwrap() == 10 + result[0].unwrap() == Channel.STOP and: - result[1].val == 20 - result[1].val == 30 - result[1].val == Channel.STOP + result[1].unwrap() == 20 + result[1].unwrap() == 30 + result[1].unwrap() == Channel.STOP } @@ -93,12 +93,12 @@ class BranchOpTest extends Dsl2Spec { then: result.size() == 2 and: - result[0].val == 1 - result[0].val == 2 - result[0].val == 3 - result[0].val == Channel.STOP + result[0].unwrap() == 1 + result[0].unwrap() == 2 + result[0].unwrap() == 3 + result[0].unwrap() == Channel.STOP and: - result[1].val == Channel.STOP + result[1].unwrap() == Channel.STOP } @@ -164,14 +164,14 @@ class BranchOpTest extends Dsl2Spec { then: result.size() == 3 and: - result[0].val == 0 - result[0].val == Channel.STOP + result[0].unwrap() == 0 + result[0].unwrap() == Channel.STOP and: - result[1].val == 10 - result[1].val == Channel.STOP + result[1].unwrap() == 10 + result[1].unwrap() == Channel.STOP and: - result[2].val == 20 - result[2].val == Channel.STOP + result[2].unwrap() == 20 + result[2].unwrap() == Channel.STOP } def 'should handle complex nested return statement' () { @@ -187,10 +187,10 @@ class BranchOpTest extends Dsl2Spec { } ''') then: - result.val == 'less than zero' - result.val == 'zero' - result.val == 'great than zero' - result.val == Channel.STOP + result.unwrap() == 'less than zero' + result.unwrap() == 'zero' + result.unwrap() == 'great than zero' + result.unwrap() == Channel.STOP } @Ignore // this is not supported and require explicit use of `return` @@ -207,10 +207,10 @@ class BranchOpTest extends Dsl2Spec { } ''') then: - result.val == 'less than zero' - result.val == 'zero' - result.val == 'great than zero' - result.val == Channel.STOP + result.unwrap() == 'less than zero' + result.unwrap() == 'zero' + result.unwrap() == 'great than zero' + result.unwrap() == Channel.STOP } @@ -230,14 +230,14 @@ class BranchOpTest extends Dsl2Spec { then: result.size() == 3 and: - result[0].val == 0 - result[0].val == Channel.STOP + result[0].unwrap() == 0 + result[0].unwrap() == Channel.STOP and: - result[1].val == 3 - result[1].val == Channel.STOP + result[1].unwrap() == 3 + result[1].unwrap() == Channel.STOP and: - result[2].val == 6 - result[2].val == Channel.STOP + result[2].unwrap() == 6 + result[2].unwrap() == Channel.STOP } def 'should branch on pair argument' () { @@ -254,11 +254,11 @@ class BranchOpTest extends Dsl2Spec { then: result.size() == 2 and: - result[0].val == 1 - result[0].val == Channel.STOP + result[0].unwrap() == 1 + result[0].unwrap() == Channel.STOP and: - result[1].val == ['b', 2] - result[1].val == Channel.STOP + result[1].unwrap() == ['b', 2] + result[1].unwrap() == Channel.STOP } def 'should pass criteria as argument' () { @@ -362,10 +362,10 @@ class BranchOpTest extends Dsl2Spec { result.size() == 2 and: result[0] instanceof DataflowVariable - result[0].val == Channel.STOP + result[0].unwrap() == Channel.STOP and: result[1] instanceof DataflowVariable - result[1].val == 10 + result[1].unwrap() == 10 } } diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/BufferOpTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/BufferOpTest.groovy index 3d8494277a..00db3c1d4d 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/BufferOpTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/BufferOpTest.groovy @@ -39,16 +39,16 @@ class BufferOpTest extends Specification { when: def r1 = Channel.of(1,2,3,1,2,3).buffer({ it == 2 }) then: - r1.val == [1,2] - r1.val == [3,1,2] - r1.val == Channel.STOP + r1.unwrap() == [1,2] + r1.unwrap() == [3,1,2] + r1.unwrap() == Channel.STOP when: def r2 = Channel.of('a','b','c','a','b','z').buffer(~/b/) then: - r2.val == ['a','b'] - r2.val == ['c','a','b'] - r2.val == Channel.STOP + r2.unwrap() == ['a','b'] + r2.unwrap() == ['c','a','b'] + r2.unwrap() == Channel.STOP } @@ -57,35 +57,35 @@ class BufferOpTest extends Specification { when: def r1 = Channel.of(1,2,3,1,2,3,1).buffer( size:2 ) then: - r1.val == [1,2] - r1.val == [3,1] - r1.val == [2,3] - r1.val == Channel.STOP + r1.unwrap() == [1,2] + r1.unwrap() == [3,1] + r1.unwrap() == [2,3] + r1.unwrap() == Channel.STOP when: r1 = Channel.of(1,2,3,1,2,3,1).buffer( size:2, remainder: true ) then: - r1.val == [1,2] - r1.val == [3,1] - r1.val == [2,3] - r1.val == [1] - r1.val == Channel.STOP + r1.unwrap() == [1,2] + r1.unwrap() == [3,1] + r1.unwrap() == [2,3] + r1.unwrap() == [1] + r1.unwrap() == Channel.STOP when: def r2 = Channel.of(1,2,3,4,5,1,2,3,4,5,1,2,9).buffer( size:3, skip:2 ) then: - r2.val == [3,4,5] - r2.val == [3,4,5] - r2.val == Channel.STOP + r2.unwrap() == [3,4,5] + r2.unwrap() == [3,4,5] + r2.unwrap() == Channel.STOP when: r2 = Channel.of(1,2,3,4,5,1,2,3,4,5,1,2,9).buffer( size:3, skip:2, remainder: true ) then: - r2.val == [3,4,5] - r2.val == [3,4,5] - r2.val == [9] - r2.val == Channel.STOP + r2.unwrap() == [3,4,5] + r2.unwrap() == [3,4,5] + r2.unwrap() == [9] + r2.unwrap() == Channel.STOP } @@ -105,16 +105,16 @@ class BufferOpTest extends Specification { when: def r1 = Channel.of(1,2,3,4,5,1,2,3,4,5,1,2).buffer( 2, 4 ) then: - r1.val == [2,3,4] - r1.val == [2,3,4] - r1.val == Channel.STOP + r1.unwrap() == [2,3,4] + r1.unwrap() == [2,3,4] + r1.unwrap() == Channel.STOP when: def r2 = Channel.of('a','b','c','a','b','z').buffer(~/a/,~/b/) then: - r2.val == ['a','b'] - r2.val == ['a','b'] - r2.val == Channel.STOP + r2.unwrap() == ['a','b'] + r2.unwrap() == ['a','b'] + r2.unwrap() == Channel.STOP } @@ -124,9 +124,9 @@ class BufferOpTest extends Specification { def sum = 0 def r1 = Channel.of(1,2,3,1,2,3).buffer(remainder: true, { sum+=it; sum==7 }) then: - r1.val == [1,2,3,1] - r1.val == [2,3] - r1.val == Channel.STOP + r1.unwrap() == [1,2,3,1] + r1.unwrap() == [2,3] + r1.unwrap() == Channel.STOP } @@ -135,19 +135,19 @@ class BufferOpTest extends Specification { when: def result = Channel.value(1).buffer(size: 1) then: - result.val == [1] - result.val == Channel.STOP + result.unwrap() == [1] + result.unwrap() == Channel.STOP when: result = Channel.value(1).buffer(size: 10) then: - result.val == Channel.STOP + result.unwrap() == Channel.STOP when: result = Channel.value(1).buffer(size: 10,remainder: true) - result.val == [1] + result.unwrap() == [1] then: - result.val == Channel.STOP + result.unwrap() == Channel.STOP } diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/CollectFileOperatorTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/CollectFileOperatorTest.groovy index e0a6a39ad5..9b14839ac3 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/CollectFileOperatorTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/CollectFileOperatorTest.groovy @@ -46,14 +46,13 @@ class CollectFileOperatorTest extends Specification { } def testCollectFileString() { - when: def result = Channel .from('alpha','beta','gamma') .collectFile { it == 'beta' ? ['file2', it.reverse() ] : ['file1',it] } .toSortedList { it.name } - List list = result.val + List list = result.unwrap() then: list[0].name == 'file1' @@ -61,12 +60,9 @@ class CollectFileOperatorTest extends Specification { list[1].name == 'file2' list[1].text == 'ateb' - } - def testCollectFileWithFiles() { - given: def file1 = Files.createTempDirectory('temp').resolve('A') file1.deleteOnExit() @@ -85,7 +81,7 @@ class CollectFileOperatorTest extends Specification { .from(file1,file2,file3) .collectFile(sort:'index') .toSortedList { it.name } - .getVal() as List + .unwrap() as List then: list[0].name == 'A' @@ -100,7 +96,7 @@ class CollectFileOperatorTest extends Specification { .from(file1,file2,file3) .collectFile(sort:'index', newLine:true) .toSortedList { it.name } - .getVal() as List + .unwrap() as List then: list[0].name == 'A' @@ -108,19 +104,15 @@ class CollectFileOperatorTest extends Specification { list[1].name == 'B' list[1].text == 'Hello\nworld\n' - - } def testCollectManyFiles() { - - when: def list = Channel .from('Hola', 'Ciao', 'Hello', 'Bonjour', 'Halo') .collectFile(sort:'index') { item -> [ "${item[0]}.txt", item + '\n' ] } .toList() - .getVal() + .unwrap() .sort { it.name } then: @@ -130,73 +122,65 @@ class CollectFileOperatorTest extends Specification { list[1].name == 'C.txt' list[2].name == 'H.txt' list[2].text == 'Hola\nHello\nHalo\n' - } - def testCollectFileWithStrings() { - when: def result = Channel .from('alpha', 'beta', 'gamma') .collectFile(name: 'hello.txt', newLine: true, sort:'index') - def file = result.val + def file = result.unwrap() then: - result.val == Channel.STOP + result.unwrap() == Channel.STOP file.name == 'hello.txt' file.text == 'alpha\nbeta\ngamma\n' } def testCollectFileWithDefaultName() { - when: def result = Channel .from('alpha', 'beta', 'gamma') .collectFile(newLine: true, sort:'index') - def file = result.val + def file = result.unwrap() then: - result.val == Channel.STOP + result.unwrap() == Channel.STOP file.name.startsWith('collect') file.text == 'alpha\nbeta\ngamma\n' } def testCollectFileAndSortWithClosure() { - when: def result = Channel .from('delta', 'beta', 'gamma','alpha') .collectFile(newLine: true, sort:{ it -> it }) - def file = result.val + def file = result.unwrap() then: - result.val == Channel.STOP + result.unwrap() == Channel.STOP file.name.startsWith('collect') file.text == 'alpha\nbeta\ndelta\ngamma\n' } def testCollectFileAndSortWithComparator() { - when: def result = Channel .from('delta', 'beta', 'gamma','alpha') .collectFile(newLine: true, sort:{ a,b -> b<=>a } as Comparator) - def file = result.val + def file = result.unwrap() then: - result.val == Channel.STOP + result.unwrap() == Channel.STOP file.name.startsWith('collect') file.text == 'gamma\ndelta\nbeta\nalpha\n' } - def 'should collect file and skip header line' () { - given: def file1 = Files.createTempDirectory('temp').resolve('A') file1.deleteOnExit() @@ -210,13 +194,12 @@ class CollectFileOperatorTest extends Specification { file3.deleteOnExit() file3.text = 'HEADER\nxxx\nyyy\nzzz\n' - when: def files = Channel .from(file1,file2,file3) .collectFile(skip:1, sort: 'index') .toList() - .getVal() + .unwrap() def result = [:]; files.each{ result[it.name]=it } then: @@ -226,13 +209,12 @@ class CollectFileOperatorTest extends Specification { result.B.name == 'B' result.B.text == 'Hello\nworld\n' - when: files = Channel .from(file1,file2,file3) .collectFile(skip:2, sort: 'index') .toList() - .getVal() + .unwrap() result = [:]; files.each{ result[it.name]=it } then: @@ -242,13 +224,12 @@ class CollectFileOperatorTest extends Specification { result.B.name == 'B' result.B.text == 'world\n' - when: files = Channel .from(file1,file2,file3) .collectFile(skip:3, sort: 'index') .toList() - .getVal() + .unwrap() result = [:]; files.each{ result[it.name]=it } then: @@ -258,13 +239,12 @@ class CollectFileOperatorTest extends Specification { result.B.name == 'B' result.B.text == '' - when: files = Channel .from(file1,file2,file3) .collectFile(skip:10, sort: 'index') .toList() - .getVal() + .unwrap() result = [:]; files.each{ result[it.name]=it } then: @@ -273,11 +253,9 @@ class CollectFileOperatorTest extends Specification { result.B.name == 'B' result.B.text == '' - } def 'should collect file and keep header line' () { - given: def file1 = Files.createTempDirectory('temp').resolve('A') file1.deleteOnExit() @@ -291,13 +269,12 @@ class CollectFileOperatorTest extends Specification { file3.deleteOnExit() file3.text = 'HEADER\nxxx\nyyy\nzzz\n' - when: def files = Channel .from(file1,file2,file3) .collectFile(keepHeader:true, sort: 'index') .toList() - .getVal() + .unwrap() def result = [:]; files.each{ result[it.name]=it } then: @@ -306,7 +283,6 @@ class CollectFileOperatorTest extends Specification { result.B.name == 'B' result.B.text == '## HEAD ##\nHello\nworld\n' - } def 'check invalid options' () { @@ -339,6 +315,5 @@ class CollectFileOperatorTest extends Specification { new CollectFileOp(Mock(DataflowReadChannel), [seed: 'foo', keepHeader: true]) then: thrown(IllegalArgumentException) - } } diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/CollectOpTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/CollectOpTest.groovy index 84ca73d2be..2192f32ea8 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/CollectOpTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/CollectOpTest.groovy @@ -38,14 +38,14 @@ class CollectOpTest extends Specification { def source = Channel.of(1,2,3) def result = source.collect() then: - result.val == [1,2,3] - result.val instanceof ArrayBag + result.unwrap() == [1,2,3] + result.unwrap() instanceof ArrayBag when: source = Channel.empty() result = source.collect() then: - result.val == Channel.STOP + result.unwrap() == Channel.STOP } @@ -56,64 +56,59 @@ class CollectOpTest extends Specification { when: def result = source.channel().collect { it.length() } then: - result.val == [5, 4, 7] - result.val instanceof ArrayBag + result.unwrap() == [5, 4, 7] + result.unwrap() instanceof ArrayBag } @Timeout(1) def 'should collect and flatten items'() { - given: def source = [[1,['a','b']], [3,['c','d']], [5,['p','q']]] when: def result = source.channel().collect() then: - result.val == [1,['a','b'],3,['c','d'],5,['p','q']] - result.val instanceof ArrayBag + result.unwrap() == [1,['a','b'],3,['c','d'],5,['p','q']] + result.unwrap() instanceof ArrayBag when: result = source.channel().collect(flat: true) then: - result.val == [1,['a','b'],3,['c','d'],5,['p','q']] - result.val instanceof ArrayBag + result.unwrap() == [1,['a','b'],3,['c','d'],5,['p','q']] + result.unwrap() instanceof ArrayBag when: result = source.channel().collect(flat: false) then: - result.val == [[1,['a','b']], [3,['c','d']], [5,['p','q']]] - result.val instanceof ArrayBag + result.unwrap() == [[1,['a','b']], [3,['c','d']], [5,['p','q']]] + result.unwrap() instanceof ArrayBag when: result = source.channel().collect { it.flatten() } then: - result.val == [1,'a','b',3,'c','d',5,'p','q'] - result.val instanceof ArrayBag - + result.unwrap() == [1,'a','b',3,'c','d',5,'p','q'] + result.unwrap() instanceof ArrayBag } @Timeout(1) def 'should collect items into a sorted list '() { - when: def result = [3,1,4,2].channel().collect(sort: true) then: - result.val == [1,2,3,4] - result.val instanceof ArrayBag + result.unwrap() == [1,2,3,4] + result.unwrap() instanceof ArrayBag when: result = ['aaa','bb', 'c'].channel().collect(sort: {it->it.size()} as Closure) then: - result.val == ['c','bb','aaa'] - result.val instanceof ArrayBag + result.unwrap() == ['c','bb','aaa'] + result.unwrap() instanceof ArrayBag when: result = ['aaa','bb', 'c'].channel().collect(sort: {a,b -> a.size()<=>b.size()} as Comparator) then: - result.val == ['c','bb','aaa'] - result.val instanceof ArrayBag - + result.unwrap() == ['c','bb','aaa'] + result.unwrap() instanceof ArrayBag } - } diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/CombineOpTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/CombineOpTest.groovy index 2efe65faed..96625c6c09 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/CombineOpTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/CombineOpTest.groovy @@ -31,7 +31,6 @@ class CombineOpTest extends Specification { new Session() } - def 'should make a tuple' () { given: def op = new CombineOp(Mock(DataflowQueue), Mock(DataflowQueue)) @@ -63,7 +62,7 @@ class CombineOpTest extends Specification { when: def result = op.apply() - def all = (List) ToListOp.apply(result).val + def all = (List) ToListOp.apply(result).unwrap() then: all.size() == 9 ['a', 1] in all @@ -86,14 +85,13 @@ class CombineOpTest extends Specification { def op = new CombineOp(left,right) op.setPivot([0]) def result = op.apply() - def all = (List) ToListOp.apply(result).val + def all = (List) ToListOp.apply(result).unwrap() println all then: true } def 'should combine a channel with a list' () { - given: def left = Channel.of('a','b') def right = [1,2,3,4] @@ -101,7 +99,7 @@ class CombineOpTest extends Specification { when: def result = op.apply() - def all = (List) ToListOp.apply(result).val + def all = (List) ToListOp.apply(result).unwrap() then: all.size() == 8 ['a', 1] in all @@ -115,7 +113,6 @@ class CombineOpTest extends Specification { } def 'should combine a value with a list' () { - given: def left = Channel.value('x') def right = [1,2,3,4] @@ -123,7 +120,7 @@ class CombineOpTest extends Specification { when: def result = op.apply() - def all = (List) ToListOp.apply(result).val + def all = (List) ToListOp.apply(result).unwrap() then: all.size() == 4 ['x', 1] in all @@ -133,7 +130,6 @@ class CombineOpTest extends Specification { } def 'should combine two values' () { - given: def left = Channel.value('x') def right = Channel.value('z') @@ -141,7 +137,7 @@ class CombineOpTest extends Specification { when: def result = op.apply() - def all = (List) ToListOp.apply(result).val + def all = (List) ToListOp.apply(result).unwrap() then: all.size() == 1 ['x', 'z'] in all @@ -155,11 +151,10 @@ class CombineOpTest extends Specification { when: def result = op.apply() then: - result.val == Channel.STOP + result.unwrap() == Channel.STOP } def 'should chain combine ops flat default' () { - given: def ch2 = Channel.of('a','b','c') def ch3 = Channel.of('x','y') @@ -167,7 +162,7 @@ class CombineOpTest extends Specification { when: def result = new CombineOp(new CombineOp(ch1, ch2).apply(), ch3).apply() - def all = (List) ToListOp.apply(result).val + def all = (List) ToListOp.apply(result).unwrap() then: all.size() == 12 @@ -184,11 +179,9 @@ class CombineOpTest extends Specification { [2,'b','y'] in all [2,'c','x'] in all [2,'c','y'] in all - } def 'should chain combine ops flat true' () { - given: def ch1 = Channel.of(1,2) def ch2 = Channel.of('a','b','c') @@ -196,7 +189,7 @@ class CombineOpTest extends Specification { when: def result = new CombineOp(new CombineOp(ch1, ch2).apply(), ch3).apply() - def all = (List) ToListOp.apply(result).val + def all = (List) ToListOp.apply(result).unwrap() then: all.size() == 12 @@ -216,13 +209,12 @@ class CombineOpTest extends Specification { } def 'should combine with tuples' () { - when: def left = Channel.of([1, 'x'], [2,'y'], [3, 'z']) def right = ['alpha','beta','gamma'] def result = new CombineOp(left, right).apply() - def all = (List) ToListOp.apply(result).val + def all = (List) ToListOp.apply(result).unwrap() then: all.size() == 9 @@ -238,7 +230,6 @@ class CombineOpTest extends Specification { [3, 'z', 'alpha'] in all [3, 'z', 'beta'] in all [3, 'z', 'gamma'] in all - } def 'should combine with map' () { @@ -247,7 +238,7 @@ class CombineOpTest extends Specification { def left = Channel.of([id:1, val:'x'], [id:2,val:'y'], [id:3, val:'z']) def right = ['alpha','beta','gamma'] def result = left.combine(right) - def all = (List) ToListOp.apply(result).val + def all = (List) ToListOp.apply(result).unwrap() then: all.size() == 9 @@ -262,17 +253,13 @@ class CombineOpTest extends Specification { [[id:3, val:'z'], 'alpha'] in all [[id:3, val:'z'], 'beta'] in all [[id:3, val:'z'], 'gamma'] in all - } - - def 'should combine items'() { - when: def left = Channel.of(1,2,3) def right = ['a','b'] - def result = left.combine(right).toSortedList().val.iterator() + def result = left.combine(right).toSortedList().unwrap().iterator() then: result.next() == [1, 'a'] result.next() == [1, 'b'] @@ -284,7 +271,7 @@ class CombineOpTest extends Specification { when: left = Channel.of(1,2) right = Channel.of('a','b','c') - result = left.combine(right).toSortedList().val.iterator() + result = left.combine(right).toSortedList().unwrap().iterator() then: result.next() == [1, 'a'] result.next() == [1, 'b'] @@ -292,15 +279,13 @@ class CombineOpTest extends Specification { result.next() == [2, 'a'] result.next() == [2, 'b'] result.next() == [2, 'c'] - } def 'should chain combine'() { - when: def str1 = Channel.of('a','b','c') def str2 = Channel.of('x','y') - def result = Channel.of(1,2).combine(str1).combine(str2).toSortedList().val.iterator() + def result = Channel.of(1,2).combine(str1).combine(str2).toSortedList().unwrap().iterator() then: result.next() == [1,'a','x'] result.next() == [1,'a','y'] @@ -318,7 +303,7 @@ class CombineOpTest extends Specification { when: str1 = Channel.of('a','b','c') str2 = Channel.of('x','y') - result = Channel.of(1,2).combine(str1).combine(str2,flat:false).toSortedList().val.iterator() + result = Channel.of(1,2).combine(str1).combine(str2,flat:false).toSortedList().unwrap().iterator() then: result.next() == [1,'a','x'] result.next() == [1,'a','y'] @@ -335,7 +320,6 @@ class CombineOpTest extends Specification { } def 'should combine by first element' () { - given: def left = Channel.of( ['A',1], ['A',2], ['B',1], ['B',2] ) def right = Channel.of( ['A',1], ['A',2], ['B',1], ['B',2] ) @@ -344,7 +328,7 @@ class CombineOpTest extends Specification { def op = new CombineOp(left, right) op.pivot = 0 def result = op.apply() - def all = (List) ToListOp.apply(result).val + def all = (List) ToListOp.apply(result).unwrap() then: all.size() == 8 @@ -367,7 +351,7 @@ class CombineOpTest extends Specification { when: def result = left.channel().combine(right.channel()) - def all = (List) ToListOp.apply(result).val + def all = (List) ToListOp.apply(result).unwrap() then: all.size() == 9 [1, 'a', 2, 'p'] in all @@ -384,7 +368,7 @@ class CombineOpTest extends Specification { when: result = left.channel().combine(right.channel(), by: 0) - all = (List) ToListOp.apply(result).val + all = (List) ToListOp.apply(result).unwrap() then: all.size() == 4 [1, 'a', 'r'] in all @@ -394,7 +378,7 @@ class CombineOpTest extends Specification { when: result = left.channel().combine(right.channel(), by: [0]) - all = (List) ToListOp.apply(result).val + all = (List) ToListOp.apply(result).unwrap() then: all.size() == 4 [1, 'a', 'r'] in all diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/ConcatOpTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/ConcatOpTest.groovy index 1600300d97..1941932bb8 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/ConcatOpTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/ConcatOpTest.groovy @@ -36,13 +36,13 @@ class ConcatOpTest extends Dsl2Spec { c1.concat(c2) ''') then: - result.val == 1 - result.val == 2 - result.val == 3 - result.val == 'a' - result.val == 'b' - result.val == 'c' - result.val == Channel.STOP + result.unwrap() == 1 + result.unwrap() == 2 + result.unwrap() == 3 + result.unwrap() == 'a' + result.unwrap() == 'b' + result.unwrap() == 'c' + result.unwrap() == Channel.STOP } def 'should concat value with channel'() { @@ -53,10 +53,10 @@ class ConcatOpTest extends Dsl2Spec { ch1.concat(ch2) ''') then: - result.val == 1 - result.val == 2 - result.val == 3 - result.val == Channel.STOP + result.unwrap() == 1 + result.unwrap() == 2 + result.unwrap() == 3 + result.unwrap() == Channel.STOP } def 'should concat two value channels'() { @@ -67,9 +67,9 @@ class ConcatOpTest extends Dsl2Spec { ch1.concat(ch2) ''') then: - result.val == 1 - result.val == 2 - result.val == Channel.STOP + result.unwrap() == 1 + result.unwrap() == 2 + result.unwrap() == Channel.STOP } def 'should concat with empty'() { @@ -80,8 +80,8 @@ class ConcatOpTest extends Dsl2Spec { ch1.concat(ch2) ''') then: - result.val == 1 - result.val == Channel.STOP + result.unwrap() == 1 + result.unwrap() == Channel.STOP when: result = dsl_eval(''' @@ -90,7 +90,7 @@ class ConcatOpTest extends Dsl2Spec { ch1.concat(ch2) ''') then: - result.val == Channel.STOP + result.unwrap() == Channel.STOP } } diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/CountFastaOpTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/CountFastaOpTest.groovy index 5efe023050..2f6c9fcfd9 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/CountFastaOpTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/CountFastaOpTest.groovy @@ -28,7 +28,6 @@ import test.TestHelper class CountFastaOpTest extends Specification { def 'should count fasta channel' () { - given: def str = ''' >1aboA @@ -64,8 +63,7 @@ class CountFastaOpTest extends Specification { when: def result = Channel.of( str, str2 ).countFasta() then: - result.val == 8 - + result.unwrap() == 8 } def 'should count fasta records from files' () { @@ -113,7 +111,6 @@ class CountFastaOpTest extends Specification { when: def result = Channel.of( file1, file2 ).countFasta() then: - result.val == 10 - + result.unwrap() == 10 } } diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/DataflowMathExtensionTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/DataflowMathExtensionTest.groovy index afff091b8a..3e076c568e 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/DataflowMathExtensionTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/DataflowMathExtensionTest.groovy @@ -34,52 +34,47 @@ class DataflowMathExtensionTest extends Specification { Comparator makeComparator(Closure c) { c as Comparator } def 'should return the min value'() { - expect: - Channel.of(4,1,7,5).min().val == 1 - Channel.of("hello","hi","hey").min { it.size() } .val == "hi" - Channel.of("hello","hi","hey").min { a,b -> a.size()<=>b.size() } .val == "hi" - Channel.of("hello","hi","hey").min { a,b -> a.size()<=>b.size() } .val == "hi" - Channel.of("hello","hi","hey").min ( makeComparator({ a,b -> a.size()<=>b.size() }) ) .val == "hi" - + Channel.of(4,1,7,5).min().unwrap() == 1 + Channel.of("hello","hi","hey").min { it.size() } .unwrap() == "hi" + Channel.of("hello","hi","hey").min { a,b -> a.size()<=>b.size() } .unwrap() == "hi" + Channel.of("hello","hi","hey").min { a,b -> a.size()<=>b.size() } .unwrap() == "hi" + Channel.of("hello","hi","hey").min ( makeComparator({ a,b -> a.size()<=>b.size() }) ) .unwrap() == "hi" } def 'should return the max value'() { expect: - Channel.of(4,1,7,5).max().val == 7 - Channel.of("hello","hi","hey").max { it.size() } .val == "hello" - Channel.of("hello","hi","hey").max { a,b -> a.size()<=>b.size() } .val == "hello" - Channel.of("hello","hi","hey").max { a,b -> a.size()<=>b.size() } .val == "hello" + Channel.of(4,1,7,5).max().unwrap() == 7 + Channel.of("hello","hi","hey").max { it.size() } .unwrap() == "hello" + Channel.of("hello","hi","hey").max { a,b -> a.size()<=>b.size() } .unwrap() == "hello" + Channel.of("hello","hi","hey").max { a,b -> a.size()<=>b.size() } .unwrap() == "hello" // this may fail randomly - the cause should be investigated - Channel.of("hello","hi","hey").max (makeComparator({ a,b -> a.size()<=>b.size() })) .val == "hello" - + Channel.of("hello","hi","hey").max (makeComparator({ a,b -> a.size()<=>b.size() })) .unwrap() == "hello" } def 'should return the sum'() { expect: - Channel.of(4,1,7,5).sum().val == 17 - Channel.of(4,1,7,5).sum { it * 2 } .val == 34 - Channel.of( [1,1,1], [0,1,2], [10,20,30] ). sum() .val == [ 11, 22, 33 ] + Channel.of(4,1,7,5).sum().unwrap() == 17 + Channel.of(4,1,7,5).sum { it * 2 } .unwrap() == 34 + Channel.of( [1,1,1], [0,1,2], [10,20,30] ). sum() .unwrap() == [ 11, 22, 33 ] } - def 'should return the mean'() { expect: - Channel.of(10,20,30).mean().val == 20 - Channel.of(10,20,30).mean { it * 2 }.val == 40 - Channel.of( [10,20,30], [10, 10, 10 ], [10, 30, 50]).mean().val == [10, 20, 30] + Channel.of(10,20,30).mean().unwrap() == 20 + Channel.of(10,20,30).mean { it * 2 }.unwrap() == 40 + Channel.of( [10,20,30], [10, 10, 10 ], [10, 30, 50]).mean().unwrap() == [10, 20, 30] } def 'should convert string to integers' () { - expect: - Channel.value('11').toInteger().val == 11 + Channel.value('11').toInteger().unwrap() == 11 when: def list = Channel.of('1', '4\n', ' 7 ', '100' ) .toInteger() .toList() - .getVal() + .unwrap() then: list.size() == 4 @@ -93,17 +88,15 @@ class DataflowMathExtensionTest extends Specification { list[3] instanceof Integer } - def 'should convert string to long' () { - expect: - Channel.value('33').toLong().val == 33L + Channel.value('33').toLong().unwrap() == 33L when: def list = Channel.of('1', '4\n', ' 7 ', '100' ) .toLong() .toList() - .getVal() + .unwrap() then: list.size() == 4 @@ -118,16 +111,14 @@ class DataflowMathExtensionTest extends Specification { } def 'should convert string to float' () { - - expect: - Channel.value('99.1').toFloat().val == 99.1f + Channel.value('99.1').toFloat().unwrap() == 99.1f when: def list = Channel.of('1', '4\n', ' 7.5 ', '100.1' ) .toFloat() .toList() - .getVal() + .unwrap() then: list.size() == 4 @@ -142,15 +133,14 @@ class DataflowMathExtensionTest extends Specification { } def 'should convert string to double' () { - expect: - Channel.value('99.1').toDouble().val == 99.1d + Channel.value('99.1').toDouble().unwrap() == 99.1d when: def list = Channel.of('1', '4\n', ' 7.5 ', '100.1' ) .toDouble() .toList() - .getVal() + .unwrap() then: list.size() == 4 @@ -162,17 +152,15 @@ class DataflowMathExtensionTest extends Specification { list[1] instanceof Double list[2] instanceof Double list[3] instanceof Double - } @Retry def 'should return a random sample' () { - when: def result = Channel .of(0,1,2,3,4,5,6,7,8,9) .randomSample(5) - .toList().val as List + .toList().unwrap() as List then: result.size() == 5 @@ -183,7 +171,6 @@ class DataflowMathExtensionTest extends Specification { result[2] in 0..9 result[3] in 0..9 result[4] in 0..9 - } } diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/DataflowMergeExtensionTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/DataflowMergeExtensionTest.groovy index 34940f48b8..b8d9db4a56 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/DataflowMergeExtensionTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/DataflowMergeExtensionTest.groovy @@ -48,10 +48,10 @@ class DataflowMergeExtensionTest extends Specification { def result = alpha.merge( beta, delta ) { a,b,c -> [c,b,a] } then: result instanceof DataflowQueue - result.val == [7,2,1] - result.val == [8,4,3] - result.val == [1,6,5] - result.val == Channel.STOP + result.unwrap() == [7,2,1] + result.unwrap() == [8,4,3] + result.unwrap() == [1,6,5] + result.unwrap() == Channel.STOP } def 'should merge with open array' () { @@ -62,10 +62,10 @@ class DataflowMergeExtensionTest extends Specification { def result = alpha.merge( beta, delta ) then: result instanceof DataflowQueue - result.val == [1,2,7] - result.val == [3,4,8] - result.val == [5,6,1] - result.val == Channel.STOP + result.unwrap() == [1,2,7] + result.unwrap() == [3,4,8] + result.unwrap() == [5,6,1] + result.unwrap() == Channel.STOP } def 'should merge with with default'() { @@ -76,10 +76,10 @@ class DataflowMergeExtensionTest extends Specification { def result = left.merge(right) then: result instanceof DataflowQueue - result.val == [1,2] - result.val == [3,4] - result.val == [5,6] - result.val == Channel.STOP + result.unwrap() == [1,2] + result.unwrap() == [3,4] + result.unwrap() == [5,6] + result.unwrap() == Channel.STOP when: left = Channel.of(1, 2, 3) @@ -87,10 +87,10 @@ class DataflowMergeExtensionTest extends Specification { result = left.merge(right) then: result instanceof DataflowQueue - result.val == [1, 'a','b'] - result.val == [2, 'p','q'] - result.val == [3, 'x','z'] - result.val == Channel.STOP + result.unwrap() == [1, 'a','b'] + result.unwrap() == [2, 'p','q'] + result.unwrap() == [3, 'x','z'] + result.unwrap() == Channel.STOP when: left = Channel.of('A','B','C') @@ -98,10 +98,10 @@ class DataflowMergeExtensionTest extends Specification { result = left.merge(right) then: result instanceof DataflowQueue - result.val == ['A', 'a', [1,2,3]] - result.val == ['B', 'b', [3,4,5]] - result.val == ['C', 'c', [6,7,8]] - result.val == Channel.STOP + result.unwrap() == ['A', 'a', [1,2,3]] + result.unwrap() == ['B', 'b', [3,4,5]] + result.unwrap() == ['C', 'c', [6,7,8]] + result.unwrap() == Channel.STOP } @@ -114,10 +114,10 @@ class DataflowMergeExtensionTest extends Specification { def result = alpha.merge( [beta, delta] ) { a,b,c -> [c,b,a] } then: result instanceof DataflowQueue - result.val == [7,2,1] - result.val == [8,4,3] - result.val == [1,6,5] - result.val == Channel.STOP + result.unwrap() == [7,2,1] + result.unwrap() == [8,4,3] + result.unwrap() == [1,6,5] + result.unwrap() == Channel.STOP } @@ -131,10 +131,10 @@ class DataflowMergeExtensionTest extends Specification { then: result instanceof DataflowQueue - result.val == [1,3] - result.val == [3,5] - result.val == [5,7] - result.val == Channel.STOP + result.unwrap() == [1,3] + result.unwrap() == [3,5] + result.unwrap() == [5,7] + result.unwrap() == Channel.STOP } def 'should merge with variables with custom closure'() { @@ -145,8 +145,8 @@ class DataflowMergeExtensionTest extends Specification { def result = alpha.merge(beta) { a,b -> [b, a] } then: result instanceof DataflowVariable - result.val == ['World', 'Hello'] - result.val == ['World', 'Hello'] + result.unwrap() == ['World', 'Hello'] + result.unwrap() == ['World', 'Hello'] } def 'should merge variables' () { @@ -156,8 +156,8 @@ class DataflowMergeExtensionTest extends Specification { def result = alpha.merge(beta) then: result instanceof DataflowVariable - result.val == ['Hello','World'] - result.val == ['Hello','World'] + result.unwrap() == ['Hello','World'] + result.unwrap() == ['Hello','World'] } } diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/DataflowTapExtensionTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/DataflowTapExtensionTest.groovy index fbd484aa3f..1459ab9486 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/DataflowTapExtensionTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/DataflowTapExtensionTest.groovy @@ -16,13 +16,11 @@ package nextflow.extension -import groovyx.gpars.dataflow.DataflowQueue -import groovyx.gpars.dataflow.DataflowVariable + import nextflow.Channel import nextflow.Session import spock.lang.Shared import spock.lang.Specification - /** * * @author Paolo Di Tommaso @@ -42,15 +40,15 @@ class DataflowTapExtensionTest extends Specification { when: def result = Channel.of( 4,7,9 ) .tap { first }.map { it+1 } then: - session.binding.first.val == 4 - session.binding.first.val == 7 - session.binding.first.val == 9 - session.binding.first.val == Channel.STOP + session.binding.first.unwrap() == 4 + session.binding.first.unwrap() == 7 + session.binding.first.unwrap() == 9 + session.binding.first.unwrap() == Channel.STOP - result.val == 5 - result.val == 8 - result.val == 10 - result.val == Channel.STOP + result.unwrap() == 5 + result.unwrap() == 8 + result.unwrap() == 10 + result.unwrap() == Channel.STOP !session.dag.isEmpty() @@ -61,19 +59,19 @@ class DataflowTapExtensionTest extends Specification { when: def result = Channel.of( 4,7,9 ) .tap { foo; bar }.map { it+1 } then: - session.binding.foo.val == 4 - session.binding.foo.val == 7 - session.binding.foo.val == 9 - session.binding.foo.val == Channel.STOP - session.binding.bar.val == 4 - session.binding.bar.val == 7 - session.binding.bar.val == 9 - session.binding.bar.val == Channel.STOP - - result.val == 5 - result.val == 8 - result.val == 10 - result.val == Channel.STOP + session.binding.foo.unwrap() == 4 + session.binding.foo.unwrap() == 7 + session.binding.foo.unwrap() == 9 + session.binding.foo.unwrap() == Channel.STOP + session.binding.bar.unwrap() == 4 + session.binding.bar.unwrap() == 7 + session.binding.bar.unwrap() == 9 + session.binding.bar.unwrap() == Channel.STOP + + result.unwrap() == 5 + result.unwrap() == 8 + result.unwrap() == 10 + result.unwrap() == Channel.STOP !session.dag.isEmpty() diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/JoinOpTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/JoinOpTest.groovy index 673686fdf9..34c4640231 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/JoinOpTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/JoinOpTest.groovy @@ -41,7 +41,7 @@ class JoinOpTest extends Specification { when: def op = new JoinOp(ch1, ch2) - def result = op.apply().toList().getVal() + def result = op.apply().toList().unwrap() then: result.size() == 3 result.contains( ['X', 1, 4] ) @@ -50,7 +50,6 @@ class JoinOpTest extends Specification { } - def 'should join entries by index' () { given: def ch1 = Channel.of([1, 'X'], [2, 'Y'], [3, 'Z'], [7, 'P']) @@ -58,7 +57,7 @@ class JoinOpTest extends Specification { when: def op = new JoinOp(ch1, ch2, [by:1]) - def result = op.apply().toList().getVal() + def result = op.apply().toList().unwrap() then: result.size() == 3 result.contains( ['X', 1, 4] ) @@ -73,7 +72,7 @@ class JoinOpTest extends Specification { when: def op = new JoinOp(ch1, ch2, [by:[1,2]]) - def result = op.apply().toList().getVal() + def result = op.apply().toList().unwrap() then: result.size() == 3 result.contains( ['a','b', 1, ['foo'], 4, [444]] ) @@ -90,7 +89,7 @@ class JoinOpTest extends Specification { when: def op = new JoinOp(ch1, ch2, [remainder: true]) - def result = op.apply().toList().getVal() + def result = op.apply().toList().unwrap() then: result.size() == 5 result.contains( ['X', 1, 4] ) @@ -108,7 +107,7 @@ class JoinOpTest extends Specification { when: def op = new JoinOp(ch1, ch2) - def result = op.apply().toList().getVal() + def result = op.apply().toList().unwrap() then: result.size() == 3 result == [1,2,3] @@ -122,7 +121,7 @@ class JoinOpTest extends Specification { when: def op = new JoinOp(ch1, ch2, [remainder: true]) - def result = op.apply().toList().getVal() + def result = op.apply().toList().unwrap() then: result.size() == 8 result == [1, 2, 3, 0, 0, 7, 8, 9] @@ -135,28 +134,25 @@ class JoinOpTest extends Specification { def right = Channel.empty() def result = left.join(right, remainder: true) then: - result.val == 1 - result.val == 2 - result.val == 3 - result.val == Channel.STOP - + result.unwrap() == 1 + result.unwrap() == 2 + result.unwrap() == 3 + result.unwrap() == Channel.STOP } def 'should join empty channel with pairs and remainder' () { - when: def left = Channel.of(['X', 1], ['Y', 2], ['Z', 3]) def right = Channel.empty() def result = left.join(right, remainder: true) then: - result.val == ['X', 1, null] - result.val == ['Y', 2, null] - result.val == ['Z', 3, null] - result.val == Channel.STOP + result.unwrap() == ['X', 1, null] + result.unwrap() == ['Y', 2, null] + result.unwrap() == ['Z', 3, null] + result.unwrap() == Channel.STOP } def 'should join a singleton value' () { - when: given: def ch1 = Channel.of( 1,2,3 ) @@ -164,35 +160,32 @@ class JoinOpTest extends Specification { when: def op = new JoinOp(ch1, ch2) - def result = op.apply().toList().getVal() + def result = op.apply().toList().unwrap() then: result == [1] } - def 'should join pair with singleton and remainder' () { - when: def left = Channel.of(['P', 0], ['X', 1], ['Y', 2], ['Z', 3]) def right = Channel.of('X', 'Y', 'Z', 'Q') def result = left.join(right) then: - result.val == ['X', 1] - result.val == ['Y', 2] - result.val == ['Z', 3] - result.val == Channel.STOP + result.unwrap() == ['X', 1] + result.unwrap() == ['Y', 2] + result.unwrap() == ['Z', 3] + result.unwrap() == Channel.STOP when: left = Channel.of(['P', 0], ['X', 1], ['Y', 2], ['Z', 3]) right = Channel.of('X', 'Y', 'Z', 'Q') - result = left.join(right, remainder: true).toList().val.sort { it -> it[0] } + result = left.join(right, remainder: true).toList().unwrap().sort { it -> it[0] } then: result[2] == ['X', 1] result[3] == ['Y', 2] result[4] == ['Z', 3] result[0] == ['P', 0] result[1] == ['Q', null] - } def 'should match gstrings' () { @@ -201,12 +194,11 @@ class JoinOpTest extends Specification { def left = Channel.of(['A', 'hola'], ['B', 'hello'], ['C', 'ciao']) def right = Channel.of(["$A", 'mundo'], ["$B", 'world'], ["$C", 'mondo'] ) when: - def result = left.join(right).toList().val.sort { it[0] } + def result = left.join(right).toList().unwrap().sort { it[0] } then: result[0] == ['A','hola','mundo'] result[1] == ['B','hello','world'] result[2] == ['C','ciao','mondo'] - } def 'should be able to use identical ArrayBags join key' () { @@ -218,7 +210,7 @@ class JoinOpTest extends Specification { when: def op = new JoinOp(ch1 as DataflowReadChannel, ch2 as DataflowReadChannel) - List result = op.apply().toList().getVal() + List result = op.apply().toList().unwrap() then: !result.isEmpty() @@ -233,7 +225,7 @@ class JoinOpTest extends Specification { when: def op = new JoinOp(ch1 as DataflowReadChannel, ch2 as DataflowReadChannel) - List result = op.apply().toList().getVal() + List result = op.apply().toList().unwrap() then: result.isEmpty() @@ -246,7 +238,7 @@ class JoinOpTest extends Specification { when: def op = new JoinOp(ch1, ch2, [failOnMismatch:true]) - def result = op.apply().toList().getVal() + def result = op.apply().toList().unwrap() then: result.size() == 2 result.contains( ['X', 1, 6] ) @@ -262,7 +254,7 @@ class JoinOpTest extends Specification { when: def op = new JoinOp(ch1, ch2, [failOnMismatch:true]) - def result = op.apply().toList().getVal() + def result = op.apply().toList().unwrap() and: await(sess) then: @@ -305,7 +297,7 @@ class JoinOpTest extends Specification { when: def op = new JoinOp(ch1, ch2, [:]) - def result = op.apply().toList().getVal() + def result = op.apply().toList().unwrap() then: result.size() == 2 result.contains( ['X', 1, 2] ) @@ -321,7 +313,7 @@ class JoinOpTest extends Specification { when: def op = new JoinOp(ch1, ch2, [failOnDuplicate:true]) - def result = op.apply().toList().getVal() + def result = op.apply().toList().unwrap() println "result=$result" and: await(sess) @@ -340,7 +332,7 @@ class JoinOpTest extends Specification { when: def op = new JoinOp(ch1, ch2, [failOnDuplicate:true, remainder: true]) - def result = op.apply().toList().getVal() + def result = op.apply().toList().unwrap() and: await(sess) then: @@ -357,7 +349,7 @@ class JoinOpTest extends Specification { when: def op = new JoinOp(ch1, ch2, [failOnDuplicate:true]) - def result = op.apply().toList().getVal() + def result = op.apply().toList().unwrap() then: await(sess) then: @@ -365,7 +357,6 @@ class JoinOpTest extends Specification { sess.getError().message == 'Detected join operation duplicate emission on left channel -- offending element: key=X; value=3' } - protected void await(Session session) { def begin = System.currentTimeMillis() while( !session.isAborted() && System.currentTimeMillis()-begin<5_000 ) diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/MixOpTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/MixOpTest.groovy index b91765d6da..58593044d0 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/MixOpTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/MixOpTest.groovy @@ -34,7 +34,7 @@ class MixOpTest extends Dsl2Spec { c3 = Channel.value( 'z' ) c1.mix(c2,c3) - ''') .toList().val + ''') .toList().unwrap() then: 1 in result @@ -44,7 +44,6 @@ class MixOpTest extends Dsl2Spec { 'b' in result 'z' in result !('c' in result) - } def 'should mix with value channels'() { @@ -53,7 +52,7 @@ class MixOpTest extends Dsl2Spec { Channel.value(1).mix( Channel.fromList([2,3]) ) ''') then: - result.toList().val.sort() == [1,2,3] + result.toList().unwrap().sort() == [1,2,3] } def 'should mix with two singleton'() { @@ -62,7 +61,7 @@ class MixOpTest extends Dsl2Spec { Channel.value(1).mix( Channel.value(2) ) ''') then: - result.toList().val.sort() == [1,2] + result.toList().unwrap().sort() == [1,2] } } diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/OperatorImplTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/OperatorImplTest.groovy index 0c581e8c19..fbfd2adeaa 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/OperatorImplTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/OperatorImplTest.groovy @@ -23,7 +23,6 @@ import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowVariable import nextflow.Channel import nextflow.Session -import nextflow.prov.Tracker import spock.lang.Specification import spock.lang.Timeout /** @@ -37,69 +36,57 @@ class OperatorImplTest extends Specification { new Session() } - private mval(Object obj) { - if( obj instanceof DataflowReadChannel ) { - def result = obj.val - return result instanceof Tracker.Msg ? result.value : result - } - else - return obj - } - def testFilter() { - when: - def c1 = Channel.of(1,2,3,4,5).filter { it > 3 } + DataflowReadChannel c1 = Channel.of(1,2,3,4,5).filter { it > 3 } then: - mval(c1) == 4 - mval(c1) == 5 - mval(c1) == Channel.STOP + c1.unwrap() == 4 + c1.unwrap() == 5 + c1.unwrap() == Channel.STOP when: def c2 = Channel.of('hola','hello','cioa','miao').filter { it =~ /^h.*/ } then: - mval(c2) == 'hola' - mval(c2) == 'hello' - mval(c2) == Channel.STOP + c2.unwrap() == 'hola' + c2.unwrap() == 'hello' + c2.unwrap() == Channel.STOP when: def c3 = Channel.of('hola','hello','cioa','miao').filter { it ==~ /^h.*/ } then: - mval(c3) == 'hola' - mval(c3) == 'hello' - mval(c3) == Channel.STOP + c3.unwrap() == 'hola' + c3.unwrap() == 'hello' + c3.unwrap() == Channel.STOP when: def c4 = Channel.of('hola','hello','cioa','miao').filter( ~/^h.*/ ) then: - mval(c4) == 'hola' - mval(c4) == 'hello' - mval(c4) == Channel.STOP + c4.unwrap() == 'hola' + c4.unwrap() == 'hello' + c4.unwrap() == Channel.STOP when: def c5 = Channel.of('hola',1,'cioa',2,3).filter( Number ) then: - mval(c5) == 1 - mval(c5) == 2 - mval(c5) == 3 - mval(c5) == Channel.STOP + c5.unwrap() == 1 + c5.unwrap() == 2 + c5.unwrap() == 3 + c5.unwrap() == Channel.STOP expect: - mval( Channel.of(1,2,4,2,4,5,6,7,4).filter(1) .count() ) == 1 - mval( Channel.of(1,2,4,2,4,5,6,7,4).filter(2) .count() ) == 2 - mval( Channel.of(1,2,4,2,4,5,6,7,4).filter(4) .count() ) == 3 - + Channel.of(1,2,4,2,4,5,6,7,4).filter(1) .count() .unwrap() == 1 + Channel.of(1,2,4,2,4,5,6,7,4).filter(2) .count() .unwrap() == 2 + Channel.of(1,2,4,2,4,5,6,7,4).filter(4) .count() .unwrap() == 3 } def testFilterWithValue() { expect: - mval(Channel.value(3).filter { it>1 }) == 3 - mval(Channel.value(0).filter { it>1 }) == Channel.STOP - mval(Channel.value(Channel.STOP).filter { it>1 }) == Channel.STOP + Channel.value(3).filter { it>1 }.unwrap() == 3 + Channel.value(0).filter { it>1 }.unwrap() == Channel.STOP + Channel.value(Channel.STOP).filter { it>1 }.unwrap() == Channel.STOP } def testSubscribe() { - when: def channel = Channel.create() int count = 0 @@ -115,12 +102,9 @@ class OperatorImplTest extends Specification { sleep(100) then: count == 4 - - } def testSubscribe1() { - when: def count = 0 def done = false @@ -129,11 +113,9 @@ class OperatorImplTest extends Specification { then: done count == 3 - } def testSubscribe2() { - when: def count = 0 def done = false @@ -142,11 +124,9 @@ class OperatorImplTest extends Specification { then: done count == 1 - } def testSubscribeError() { - when: int next=0 int error=0 @@ -164,18 +144,16 @@ class OperatorImplTest extends Specification { error == 1 // complete never complete == 0 - } - def testMap() { when: def result = Channel.of(1,2,3).map { "Hello $it" } then: - mval(result) == 'Hello 1' - mval(result) == 'Hello 2' - mval(result) == 'Hello 3' - mval(result) == Channel.STOP + result.unwrap() == 'Hello 1' + result.unwrap() == 'Hello 2' + result.unwrap() == 'Hello 3' + result.unwrap() == Channel.STOP } def testMapWithVariable() { @@ -184,446 +162,406 @@ class OperatorImplTest extends Specification { when: def result = variable.map { it.reverse() } then: - mval(result) == 'olleH' - mval(result) == 'olleH' - mval(result) == 'olleH' + result.unwrap() == 'olleH' + result.unwrap() == 'olleH' + result.unwrap() == 'olleH' } def testMapParamExpanding () { - when: def result = Channel.of(1,2,3).map { [it, it] }.map { x, y -> x+y } then: - mval(result) == 2 - mval(result) == 4 - mval(result) == 6 - mval(result) == Channel.STOP + result.unwrap() == 2 + result.unwrap() == 4 + result.unwrap() == 6 + result.unwrap() == Channel.STOP } def testSkip() { - when: def result = Channel.of(1,2,3).map { it == 2 ? Channel.VOID : "Hello $it" } then: - mval(result) == 'Hello 1' - mval(result) == 'Hello 3' - mval(result) == Channel.STOP - + result.unwrap() == 'Hello 1' + result.unwrap() == 'Hello 3' + result.unwrap() == Channel.STOP } - def testMapMany () { - when: def result = Channel.of(1,2,3).flatMap { it -> [it, it*2] } then: - mval(result) == 1 - mval(result) == 2 - mval(result) == 2 - mval(result) == 4 - mval(result) == 3 - mval(result) == 6 - mval(result) == Channel.STOP + result.unwrap() == 1 + result.unwrap() == 2 + result.unwrap() == 2 + result.unwrap() == 4 + result.unwrap() == 3 + result.unwrap() == 6 + result.unwrap() == Channel.STOP } def testMapManyWithSingleton() { - when: def result = Channel.value([1,2,3]).flatMap() then: - mval(result) == 1 - mval(result) == 2 - mval(result) == 3 - mval(result) == Channel.STOP + result.unwrap() == 1 + result.unwrap() == 2 + result.unwrap() == 3 + result.unwrap() == Channel.STOP when: result = Channel.empty().flatMap() then: - mval(result) == Channel.STOP - + result.unwrap() == Channel.STOP } def testMapManyWithTuples () { - when: def result = Channel.of( [1,2], ['a','b'] ).flatMap { it -> [it, it.reverse()] } then: - mval(result) == [1, 2] - mval(result) == [2, 1] - mval(result) == ['a', 'b'] - mval(result) == ['b', 'a'] - mval(result) == Channel.STOP + result.unwrap() == [1, 2] + result.unwrap() == [2, 1] + result.unwrap() == ['a', 'b'] + result.unwrap() == ['b', 'a'] + result.unwrap() == Channel.STOP } def testMapManyDefault () { - when: def result = Channel.of( [1,2], ['a',['b','c']] ).flatMap() then: - mval(result) == 1 - mval(result) == 2 - mval(result) == 'a' - mval(result) == ['b', 'c'] // <-- nested list are preserved - mval(result) == Channel.STOP + result.unwrap() == 1 + result.unwrap() == 2 + result.unwrap() == 'a' + result.unwrap() == ['b', 'c'] // <-- nested list are preserved + result.unwrap() == Channel.STOP } def testMapManyWithHashArray () { - when: def result = Channel.of(1,2,3).flatMap { it -> [ k: it, v: it*2] } then: - mval(result) == new MapEntry('k',1) - mval(result) == new MapEntry('v',2) - mval(result) == new MapEntry('k',2) - mval(result) == new MapEntry('v',4) - mval(result) == new MapEntry('k',3) - mval(result) == new MapEntry('v',6) - mval(result) == Channel.STOP - + result.unwrap() == new MapEntry('k',1) + result.unwrap() == new MapEntry('v',2) + result.unwrap() == new MapEntry('k',2) + result.unwrap() == new MapEntry('v',4) + result.unwrap() == new MapEntry('k',3) + result.unwrap() == new MapEntry('v',6) + result.unwrap() == Channel.STOP } - - def testReduce() { - when: def channel = Channel.create() def result = channel.reduce { a, e -> a += e } channel << 1 << 2 << 3 << 4 << 5 << Channel.STOP then: - mval(result) == 15 - + result.unwrap() == 15 when: channel = Channel.of(1,2,3,4,5) result = channel.reduce { a, e -> a += e } then: - mval(result) == 15 + result.unwrap() == 15 when: channel = Channel.create() result = channel.reduce { a, e -> a += e } channel << 99 << Channel.STOP then: - mval(result) == 99 + result.unwrap() == 99 when: channel = Channel.create() result = channel.reduce { a, e -> a += e } channel << Channel.STOP then: - mval(result) == null + result.unwrap() == null when: result = Channel.of(6,5,4,3,2,1).reduce { a, e -> Channel.STOP } then: - mval(result) == 6 - + result.unwrap() == 6 } - def testReduceWithSeed() { - when: def channel = Channel.create() def result = channel.reduce (1) { a, e -> a += e } channel << 1 << 2 << 3 << 4 << 5 << Channel.STOP then: - mval(result) == 16 + result.unwrap() == 16 when: channel = Channel.create() result = channel.reduce (10) { a, e -> a += e } channel << Channel.STOP then: - mval(result) == 10 + result.unwrap() == 10 when: result = Channel.of(6,5,4,3,2,1).reduce(0) { a, e -> a < 3 ? a+1 : Channel.STOP } then: - mval(result) == 3 - + result.unwrap() == 3 } def testFirst() { - expect: - mval(Channel.of(3,6,4,5,4,3,4).first()) == 3 + Channel.of(3,6,4,5,4,3,4).first().unwrap() == 3 } def testFirstWithCriteria() { expect: - mval(Channel.of(3,6,4,5,4,3,4).first{ it>4 }) == 6 + Channel.of(3,6,4,5,4,3,4).first{ it>4 }.unwrap() == 6 } def testFirstWithValue() { - expect: - mval(Channel.value(3).first()) == 3 - mval(Channel.value(3).first{ it>1 }) == 3 - mval(Channel.value(3).first{ it>3 }) == Channel.STOP - mval(Channel.value(Channel.STOP).first { it>3 }) == Channel.STOP + Channel.value(3).first().unwrap() == 3 + Channel.value(3).first{ it>1 }.unwrap() == 3 + Channel.value(3).first{ it>3 }.unwrap() == Channel.STOP + Channel.value(Channel.STOP).first { it>3 }.unwrap() == Channel.STOP } - def testFirstWithCondition() { - expect: - mval(Channel.of(3,6,4,5,4,3,4).first { it % 2 == 0 }) == 6 - mval(Channel.of( 'a', 'b', 'c', 1, 2 ).first( Number )) == 1 - mval(Channel.of( 'a', 'b', 1, 2, 'aaa', 'bbb' ).first( ~/aa.*/ )) == 'aaa' - mval(Channel.of( 'a', 'b', 1, 2, 'aaa', 'bbb' ).first( 1 )) == 1 - + Channel.of(3,6,4,5,4,3,4).first { it % 2 == 0 }.unwrap() == 6 + Channel.of( 'a', 'b', 'c', 1, 2 ).first( Number ).unwrap() == 1 + Channel.of( 'a', 'b', 1, 2, 'aaa', 'bbb' ).first( ~/aa.*/ ).unwrap() == 'aaa' + Channel.of( 'a', 'b', 1, 2, 'aaa', 'bbb' ).first( 1 ).unwrap() == 1 } - def testTake() { - when: def result = Channel.of(1,2,3,4,5,6).take(3) then: - mval(result) == 1 - mval(result) == 2 - mval(result) == 3 - mval(result) == Channel.STOP + result.unwrap() == 1 + result.unwrap() == 2 + result.unwrap() == 3 + result.unwrap() == Channel.STOP when: result = Channel.of(1).take(3) then: - mval(result) == 1 - mval(result) == Channel.STOP + result.unwrap() == 1 + result.unwrap() == Channel.STOP when: result = Channel.of(1,2,3).take(0) then: - mval(result) == Channel.STOP + result.unwrap() == Channel.STOP when: result = Channel.of(1,2,3).take(-1) then: - mval(result) == 1 - mval(result) == 2 - mval(result) == 3 - mval(result) == Channel.STOP + result.unwrap() == 1 + result.unwrap() == 2 + result.unwrap() == 3 + result.unwrap() == Channel.STOP when: result = Channel.of(1,2,3).take(3) then: - mval(result) == 1 - mval(result) == 2 - mval(result) == 3 - mval(result) == Channel.STOP - + result.unwrap() == 1 + result.unwrap() == 2 + result.unwrap() == 3 + result.unwrap() == Channel.STOP } def testLast() { - expect: - mval(Channel.of(3,6,4,5,4,3,9).last()) == 9 - mval(Channel.value('x').last()) == 'x' + Channel.of(3,6,4,5,4,3,9).last().unwrap() == 9 + Channel.value('x').last().unwrap() == 'x' } - - - def testCount() { expect: - mval(Channel.of(4,1,7,5).count()) == 4 - mval(Channel.of(4,1,7,1,1).count(1)) == 3 - mval(Channel.of('a','c','c','q','b').count ( ~/c/ )) == 2 - mval(Channel.value(5).count()) == 1 - mval(Channel.value(5).count(5)) == 1 - mval(Channel.value(5).count(6)) == 0 + Channel.of(4,1,7,5).count().unwrap() == 4 + Channel.of(4,1,7,1,1).count(1).unwrap() == 3 + Channel.of('a','c','c','q','b').count ( ~/c/ ).unwrap() == 2 + Channel.value(5).count().unwrap() == 1 + Channel.value(5).count(5).unwrap() == 1 + Channel.value(5).count(6).unwrap() == 0 } def testToList() { - when: def channel = Channel.of(1,2,3) then: - mval(channel.toList()) == [1, 2, 3] + channel.toList().unwrap() == [1, 2, 3] when: channel = Channel.create() channel << Channel.STOP then: - mval(channel.toList()) == [] + channel.toList().unwrap() == [] when: channel = Channel.value(1) then: - mval(channel.toList()) == [1] + channel.toList().unwrap() == [1] when: channel = Channel.empty() then: - mval(channel.toList()) == [] + channel.toList().unwrap() == [] } def testToSortedList() { - when: def channel = Channel.of(3,1,4,2) then: - mval(channel.toSortedList()) == [1, 2, 3, 4] + channel.toSortedList().unwrap() == [1, 2, 3, 4] when: channel = Channel.empty() then: - mval(channel.toSortedList()) == [] + channel.toSortedList().unwrap() == [] when: channel = Channel.of([1,'zeta'], [2,'gamma'], [3,'alpaha'], [4,'delta']) then: - mval(channel.toSortedList { it[1] }) == [[3, 'alpaha'], [4, 'delta'], [2, 'gamma'], [1, 'zeta'] ] + channel.toSortedList { it[1] }.unwrap() == [[3, 'alpaha'], [4, 'delta'], [2, 'gamma'], [1, 'zeta'] ] when: channel = Channel.value(1) then: - mval(channel.toSortedList()) == [1] + channel.toSortedList().unwrap() == [1] when: channel = Channel.empty() then: - mval(channel.toSortedList()) == [] - + channel.toSortedList().unwrap() == [] } def testUnique() { expect: - mval(Channel.of(1,1,1,5,7,7,7,3,3).unique().toList()) == [1, 5, 7, 3] - mval(Channel.of(1,3,4,5).unique { it%2 } .toList()) == [1, 4] + Channel.of(1,1,1,5,7,7,7,3,3).unique().toList().unwrap() == [1, 5, 7, 3] + Channel.of(1,3,4,5).unique { it%2 } .toList().unwrap() == [1, 4] and: - mval(Channel.of(1).unique()) == 1 - mval(Channel.value(1).unique()) == 1 + Channel.of(1).unique().unwrap() == 1 + Channel.value(1).unique().unwrap() == 1 } def testDistinct() { expect: - mval(Channel.of(1,1,2,2,2,3,1,1,2,2,3).distinct().toList()) == [1, 2, 3, 1, 2, 3] - mval(Channel.of(1,1,2,2,2,3,1,1,2,4,6).distinct { it%2 } .toList()) == [1, 2, 3, 2] + Channel.of(1,1,2,2,2,3,1,1,2,2,3).distinct().toList().unwrap() == [1, 2, 3, 1, 2, 3] + Channel.of(1,1,2,2,2,3,1,1,2,4,6).distinct { it%2 } .toList().unwrap() == [1, 2, 3, 2] } - def testFlatten() { - when: def r1 = Channel.of(1,2,3).flatten() then: - mval(r1) == 1 - mval(r1) == 2 - mval(r1) == 3 - mval(r1) == Channel.STOP + r1.unwrap() == 1 + r1.unwrap() == 2 + r1.unwrap() == 3 + r1.unwrap() == Channel.STOP when: def r2 = Channel.of([1,'a'], [2,'b']).flatten() then: - mval(r2) == 1 - mval(r2) == 'a' - mval(r2) == 2 - mval(r2) == 'b' - mval(r2) == Channel.STOP + r2.unwrap() == 1 + r2.unwrap() == 'a' + r2.unwrap() == 2 + r2.unwrap() == 'b' + r2.unwrap() == Channel.STOP when: def r3 = Channel.of( [1,2] as Integer[], [3,4] as Integer[] ).flatten() then: - mval(r3) == 1 - mval(r3) == 2 - mval(r3) == 3 - mval(r3) == 4 - mval(r3) == Channel.STOP + r3.unwrap() == 1 + r3.unwrap() == 2 + r3.unwrap() == 3 + r3.unwrap() == 4 + r3.unwrap() == Channel.STOP when: def r4 = Channel.of( [1,[2,3]], 4, [5,[6]] ).flatten() then: - mval(r4) == 1 - mval(r4) == 2 - mval(r4) == 3 - mval(r4) == 4 - mval(r4) == 5 - mval(r4) == 6 - mval(r4) == Channel.STOP + r4.unwrap() == 1 + r4.unwrap() == 2 + r4.unwrap() == 3 + r4.unwrap() == 4 + r4.unwrap() == 5 + r4.unwrap() == 6 + r4.unwrap() == Channel.STOP } def testFlattenWithSingleton() { when: def result = Channel.value([3,2,1]).flatten() then: - mval(result) == 3 - mval(result) == 2 - mval(result) == 1 - mval(result) == Channel.STOP + result.unwrap() == 3 + result.unwrap() == 2 + result.unwrap() == 1 + result.unwrap() == Channel.STOP when: result = Channel.empty().flatten() then: - mval(result) == Channel.STOP + result.unwrap() == Channel.STOP } def testCollate() { - when: def r1 = Channel.of(1,2,3,1,2,3,1).collate( 2, false ) then: - mval(r1) == [1, 2] - mval(r1) == [3, 1] - mval(r1) == [2, 3] - mval(r1) == Channel.STOP + r1.unwrap() == [1, 2] + r1.unwrap() == [3, 1] + r1.unwrap() == [2, 3] + r1.unwrap() == Channel.STOP when: def r2 = Channel.of(1,2,3,1,2,3,1).collate( 3 ) then: - mval(r2) == [1, 2, 3] - mval(r2) == [1, 2, 3] - mval(r2) == [1] - mval(r2) == Channel.STOP - + r2.unwrap() == [1, 2, 3] + r2.unwrap() == [1, 2, 3] + r2.unwrap() == [1] + r2.unwrap() == Channel.STOP } def testCollateWithStep() { - when: def r1 = Channel.of(1,2,3,4).collate( 3, 1, false ) then: - mval(r1) == [1, 2, 3] - mval(r1) == [2, 3, 4] - mval(r1) == Channel.STOP + r1.unwrap() == [1, 2, 3] + r1.unwrap() == [2, 3, 4] + r1.unwrap() == Channel.STOP when: def r2 = Channel.of(1,2,3,4).collate( 3, 1, true ) then: - mval(r2) == [1, 2, 3] - mval(r2) == [2, 3, 4] - mval(r2) == [3, 4] - mval(r2) == [4] - mval(r2) == Channel.STOP + r2.unwrap() == [1, 2, 3] + r2.unwrap() == [2, 3, 4] + r2.unwrap() == [3, 4] + r2.unwrap() == [4] + r2.unwrap() == Channel.STOP when: def r3 = Channel.of(1,2,3,4).collate( 3, 1 ) then: - mval(r3) == [1, 2, 3] - mval(r3) == [2, 3, 4] - mval(r3) == [3, 4] - mval(r3) == [4] - mval(r3) == Channel.STOP + r3.unwrap() == [1, 2, 3] + r3.unwrap() == [2, 3, 4] + r3.unwrap() == [3, 4] + r3.unwrap() == [4] + r3.unwrap() == Channel.STOP when: def r4 = Channel.of(1,2,3,4).collate( 4,4 ) then: - mval(r4) == [1, 2, 3, 4] - mval(r4) == Channel.STOP + r4.unwrap() == [1, 2, 3, 4] + r4.unwrap() == Channel.STOP when: def r5 = Channel.of(1,2,3,4).collate( 6,6 ) then: - mval(r5) == [1, 2, 3, 4] - mval(r5) == Channel.STOP + r5.unwrap() == [1, 2, 3, 4] + r5.unwrap() == Channel.STOP when: def r6 = Channel.of(1,2,3,4).collate( 6,6,false ) then: - mval(r6) == Channel.STOP - + r6.unwrap() == Channel.STOP } def testCollateIllegalArgs() { @@ -646,32 +584,31 @@ class OperatorImplTest extends Specification { Channel.create().collate(1,0) then: thrown(IllegalArgumentException) - } def testCollateWithValueChannel() { when: def result = Channel.value(1).collate(1) then: - mval(result) == [1] - mval(result) == Channel.STOP + result.unwrap() == [1] + result.unwrap() == Channel.STOP when: result = Channel.value(1).collate(10) then: - mval(result) == [1] - mval(result) == Channel.STOP + result.unwrap() == [1] + result.unwrap() == Channel.STOP when: result = Channel.value(1).collate(10, true) then: - mval(result) == [1] - mval(result) == Channel.STOP + result.unwrap() == [1] + result.unwrap() == Channel.STOP when: result = Channel.value(1).collate(10, false) then: - mval(result) == Channel.STOP + result.unwrap() == Channel.STOP } def testMix() { @@ -679,7 +616,7 @@ class OperatorImplTest extends Specification { def c1 = Channel.of( 1,2,3 ) def c2 = Channel.of( 'a','b' ) def c3 = Channel.value( 'z' ) - def result = c1.mix(c2,c3).toList().val + def result = c1.mix(c2,c3).toList().unwrap() then: 1 in result @@ -689,20 +626,16 @@ class OperatorImplTest extends Specification { 'b' in result 'z' in result !('c' in result) - } def testMixWithSingleton() { when: def result = Channel.value(1).mix( Channel.of(2,3) ) then: - mval(result.toList().sort()) == [1, 2, 3] + result.toList().unwrap().sort() == [1, 2, 3] } - - def testDefaultMappingClosure() { - expect: OperatorImpl.DEFAULT_MAPPING_CLOSURE.call( [7, 8, 9] ) == 7 OperatorImpl.DEFAULT_MAPPING_CLOSURE.call( [7, 8, 9], 2 ) == 9 @@ -736,12 +669,10 @@ class OperatorImplTest extends Specification { OperatorImpl.DEFAULT_MAPPING_CLOSURE.call( 99 ) == 99 OperatorImpl.DEFAULT_MAPPING_CLOSURE.call( 99, 2 ) == null - } def testCross() { - setup: def ch1 = Channel.of( [1, 'x'], [2,'y'], [3,'z'] ) def ch2 = Channel.of( [1,11], [1,13], [2,21],[2,22], [2,23], [4,1], [4,2] ) @@ -750,17 +681,15 @@ class OperatorImplTest extends Specification { def result = ch1.cross(ch2) then: - mval(result) == [[1, 'x'], [1, 11] ] - mval(result) == [[1, 'x'], [1, 13] ] - mval(result) == [[2, 'y'], [2, 21] ] - mval(result) == [[2, 'y'], [2, 22] ] - mval(result) == [[2, 'y'], [2, 23] ] - mval(result) == Channel.STOP - + result.unwrap() == [[1, 'x'], [1, 11] ] + result.unwrap() == [[1, 'x'], [1, 13] ] + result.unwrap() == [[2, 'y'], [2, 21] ] + result.unwrap() == [[2, 'y'], [2, 22] ] + result.unwrap() == [[2, 'y'], [2, 23] ] + result.unwrap() == Channel.STOP } def testCross2() { - setup: def ch1 = Channel.create() def ch2 = Channel.of ( ['PF00006', 'PF00006_mafft.aln'], ['PF00006', 'PF00006_clustalo.aln']) @@ -770,15 +699,13 @@ class OperatorImplTest extends Specification { def result = ch1.cross(ch2) then: - mval(result) == [['PF00006', 'PF00006.sp_lib'], ['PF00006', 'PF00006_mafft.aln'] ] - mval(result) == [['PF00006', 'PF00006.sp_lib'], ['PF00006', 'PF00006_clustalo.aln'] ] - mval(result) == Channel.STOP - + result.unwrap() == [['PF00006', 'PF00006.sp_lib'], ['PF00006', 'PF00006_mafft.aln'] ] + result.unwrap() == [['PF00006', 'PF00006.sp_lib'], ['PF00006', 'PF00006_clustalo.aln'] ] + result.unwrap() == Channel.STOP } def testCross3() { - setup: def ch1 = Channel.of(['PF00006', 'PF00006.sp_lib']) def ch2 = Channel.create ( ) @@ -788,27 +715,25 @@ class OperatorImplTest extends Specification { def result = ch1.cross(ch2) then: - mval(result) == [['PF00006', 'PF00006.sp_lib'], ['PF00006', 'PF00006_mafft.aln'] ] - mval(result) == [['PF00006', 'PF00006.sp_lib'], ['PF00006', 'PF00006_clustalo.aln'] ] - mval(result) == Channel.STOP - + result.unwrap() == [['PF00006', 'PF00006.sp_lib'], ['PF00006', 'PF00006_mafft.aln'] ] + result.unwrap() == [['PF00006', 'PF00006.sp_lib'], ['PF00006', 'PF00006_clustalo.aln'] ] + result.unwrap() == Channel.STOP } def testConcat() { - when: def c1 = Channel.of(1,2,3) def c2 = Channel.of('a','b','c') def all = c1.concat(c2) then: - mval(all) == 1 - mval(all) == 2 - mval(all) == 3 - mval(all) == 'a' - mval(all) == 'b' - mval(all) == 'c' - mval(all) == Channel.STOP + all.unwrap() == 1 + all.unwrap() == 2 + all.unwrap() == 3 + all.unwrap() == 'a' + all.unwrap() == 'b' + all.unwrap() == 'c' + all.unwrap() == Channel.STOP when: def d1 = Channel.create() @@ -820,55 +745,50 @@ class OperatorImplTest extends Specification { Thread.start { sleep 100; d1 << 1 << 2 << Channel.STOP } then: - mval(result) == 1 - mval(result) == 2 - mval(result) == 'a' - mval(result) == 'b' - mval(result) == 'c' - mval(result) == 'p' - mval(result) == 'q' - mval(result) == Channel.STOP - + result.unwrap() == 1 + result.unwrap() == 2 + result.unwrap() == 'a' + result.unwrap() == 'b' + result.unwrap() == 'c' + result.unwrap() == 'p' + result.unwrap() == 'q' + result.unwrap() == Channel.STOP } def testContactWithSingleton() { when: def result = Channel.value(1).concat( Channel.of(2,3) ) then: - mval(result) == 1 - mval(result) == 2 - mval(result) == 3 - mval(result) == Channel.STOP + result.unwrap() == 1 + result.unwrap() == 2 + result.unwrap() == 3 + result.unwrap() == Channel.STOP } - def testGroupTuple() { - when: def result = Channel .from([1,'a'], [1,'b'], [2,'x'], [3, 'q'], [1,'c'], [2, 'y'], [3, 'q']) .groupTuple() then: - result.val == [1, ['a', 'b','c'] ] - result.val == [2, ['x', 'y'] ] - result.val == [3, ['q', 'q'] ] - result.val == Channel.STOP - + result.unwrap() == [1, ['a', 'b','c'] ] + result.unwrap() == [2, ['x', 'y'] ] + result.unwrap() == [3, ['q', 'q'] ] + result.unwrap() == Channel.STOP } def testGroupTupleWithCount() { - when: def result = Channel .from([1,'a'], [1,'b'], [2,'x'], [3, 'q'], [1,'d'], [1,'c'], [2, 'y'], [1,'f']) .groupTuple(size: 2) then: - result.val == [1, ['a', 'b'] ] - result.val == [1, ['d', 'c'] ] - result.val == [2, ['x', 'y'] ] - result.val == Channel.STOP + result.unwrap() == [1, ['a', 'b'] ] + result.unwrap() == [1, ['d', 'c'] ] + result.unwrap() == [2, ['x', 'y'] ] + result.unwrap() == Channel.STOP when: result = Channel @@ -876,27 +796,25 @@ class OperatorImplTest extends Specification { .groupTuple(size: 2, remainder: true) then: - result.val == [1, ['a', 'b'] ] - result.val == [1, ['d', 'c'] ] - result.val == [2, ['x', 'y'] ] - result.val == [3, ['q']] - result.val == [1, ['f']] - result.val == Channel.STOP - + result.unwrap() == [1, ['a', 'b'] ] + result.unwrap() == [1, ['d', 'c'] ] + result.unwrap() == [2, ['x', 'y'] ] + result.unwrap() == [3, ['q']] + result.unwrap() == [1, ['f']] + result.unwrap() == Channel.STOP } def testGroupTupleWithSortNatural() { - when: def result = Channel .from([1,'z'], [1,'w'], [1,'a'], [1,'b'], [2, 'y'], [2,'x'], [3, 'q'], [1,'c'], [3, 'p']) .groupTuple(sort: true) then: - result.val == [1, ['a', 'b','c','w','z'] ] - result.val == [2, ['x','y'] ] - result.val == [3, ['p', 'q'] ] - result.val == Channel.STOP + result.unwrap() == [1, ['a', 'b','c','w','z'] ] + result.unwrap() == [2, ['x','y'] ] + result.unwrap() == [3, ['p', 'q'] ] + result.unwrap() == Channel.STOP when: result = Channel @@ -904,77 +822,66 @@ class OperatorImplTest extends Specification { .groupTuple(sort: 'natural') then: - result.val == [1, ['a', 'b','c','w','z'] ] - result.val == [2, ['x','y'] ] - result.val == [3, ['p', 'q'] ] - result.val == Channel.STOP - + result.unwrap() == [1, ['a', 'b','c','w','z'] ] + result.unwrap() == [2, ['x','y'] ] + result.unwrap() == [3, ['p', 'q'] ] + result.unwrap() == Channel.STOP } - def testGroupTupleWithSortHash() { - when: def result = Channel .from([1,'z'], [1,'w'], [1,'a'], [1,'b'], [2, 'y'], [2,'x'], [3, 'q'], [1,'c'], [3, 'p']) .groupTuple(sort: 'hash') then: - result.val == [1, ['a', 'c','z','b','w'] ] - result.val == [2, ['y','x'] ] - result.val == [3, ['p', 'q'] ] - result.val == Channel.STOP - + result.unwrap() == [1, ['a', 'c','z','b','w'] ] + result.unwrap() == [2, ['y','x'] ] + result.unwrap() == [3, ['p', 'q'] ] + result.unwrap() == Channel.STOP } def testGroupTupleWithComparator() { - when: def result = Channel .from([1,'z'], [1,'w'], [1,'a'], [1,'b'], [2, 'y'], [2,'x'], [3, 'q'], [1,'c'], [3, 'p']) .groupTuple(sort: { o1, o2 -> o2<=>o1 } as Comparator ) then: - result.val == [1, ['z','w','c','b','a'] ] - result.val == [2, ['y','x'] ] - result.val == [3, ['q','p'] ] - result.val == Channel.STOP - + result.unwrap() == [1, ['z','w','c','b','a'] ] + result.unwrap() == [2, ['y','x'] ] + result.unwrap() == [3, ['q','p'] ] + result.unwrap() == Channel.STOP } def testGroupTupleWithClosureWithSingle() { - when: def result = Channel .from([1,'z'], [1,'w'], [1,'a'], [1,'b'], [2, 'y'], [2,'x'], [3, 'q'], [1,'c'], [3, 'p']) .groupTuple(sort: { it -> it } ) then: - result.val == [1, ['a', 'b','c','w','z'] ] - result.val == [2, ['x','y'] ] - result.val == [3, ['p', 'q'] ] - result.val == Channel.STOP - + result.unwrap() == [1, ['a', 'b','c','w','z'] ] + result.unwrap() == [2, ['x','y'] ] + result.unwrap() == [3, ['p', 'q'] ] + result.unwrap() == Channel.STOP } def testGroupTupleWithComparatorWithPair() { - when: def result = Channel .from([1,'z'], [1,'w'], [1,'a'], [1,'b'], [2, 'y'], [2,'x'], [3, 'q'], [1,'c'], [3, 'p']) .groupTuple(sort: { o1, o2 -> o2<=>o1 } ) then: - result.val == [1, ['z','w','c','b','a'] ] - result.val == [2, ['y','x'] ] - result.val == [3, ['q','p'] ] - result.val == Channel.STOP - + result.unwrap() == [1, ['z','w','c','b','a'] ] + result.unwrap() == [2, ['y','x'] ] + result.unwrap() == [3, ['q','p'] ] + result.unwrap() == Channel.STOP } def testGroupTupleWithIndex () { - given: def file1 = Paths.get('/path/file_1') def file2 = Paths.get('/path/file_2') @@ -986,10 +893,10 @@ class OperatorImplTest extends Specification { .groupTuple(by: 2) then: - result.val == [ [1,3,3], ['a','q','q'], file1 ] - result.val == [ [1,2], ['b','x'], file2 ] - result.val == [ [1,2], ['c','y'], file3 ] - result.val == Channel.STOP + result.unwrap() == [ [1,3,3], ['a','q','q'], file1 ] + result.unwrap() == [ [1,2], ['b','x'], file2 ] + result.unwrap() == [ [1,2], ['c','y'], file3 ] + result.unwrap() == Channel.STOP when: @@ -998,10 +905,10 @@ class OperatorImplTest extends Specification { .groupTuple(by: [2]) then: - result.val == [ [1,3,3], ['a','q','q'], file1 ] - result.val == [ [1,2], ['b','x'], file2 ] - result.val == [ [1,2], ['c','y'], file3 ] - result.val == Channel.STOP + result.unwrap() == [ [1,3,3], ['a','q','q'], file1 ] + result.unwrap() == [ [1,2], ['b','x'], file2 ] + result.unwrap() == [ [1,2], ['c','y'], file3 ] + result.unwrap() == Channel.STOP when: @@ -1010,17 +917,15 @@ class OperatorImplTest extends Specification { .groupTuple(by: [0,2]) then: - result.val == [ 1, ['a','q'], file1 ] - result.val == [ 1, ['b','c','z'], file2 ] - result.val == [ 2, ['x','y'], file2 ] - result.val == [ 3, ['y','c'], file3 ] - result.val == [ 3, ['q'], file1 ] - result.val == Channel.STOP - + result.unwrap() == [ 1, ['a','q'], file1 ] + result.unwrap() == [ 1, ['b','c','z'], file2 ] + result.unwrap() == [ 2, ['x','y'], file2 ] + result.unwrap() == [ 3, ['y','c'], file3 ] + result.unwrap() == [ 3, ['q'], file1 ] + result.unwrap() == Channel.STOP } def testGroupTupleWithNotMatchingCardinality() { - when: def result = Channel .of([1,'a'], @@ -1033,15 +938,13 @@ class OperatorImplTest extends Specification { .groupTuple() then: - result.val == [1, ['a', 'b', 'c'], ['d'] ] - result.val == [2, ['x', 'y'] ] - result.val == [3, ['p', 'q'] ] - result.val == Channel.STOP - + result.unwrap() == [1, ['a', 'b', 'c'], ['d'] ] + result.unwrap() == [2, ['x', 'y'] ] + result.unwrap() == [3, ['p', 'q'] ] + result.unwrap() == Channel.STOP } def testGroupTupleWithNotMatchingCardinalityAndFixedSize() { - when: def result = Channel .of([1,'a'], @@ -1054,14 +957,13 @@ class OperatorImplTest extends Specification { .groupTuple(size:2) then: - result.val == [1, ['a', 'b'] ] - result.val == [2, ['x', 'y'] ] - result.val == [3, ['p', 'q'] ] - result.val == Channel.STOP + result.unwrap() == [1, ['a', 'b'] ] + result.unwrap() == [2, ['x', 'y'] ] + result.unwrap() == [3, ['p', 'q'] ] + result.unwrap() == Channel.STOP } def testGroupTupleWithNotMatchingCardinalityAndFixedSizeAndRemainder() { - when: def result = Channel .of([1,'a'], @@ -1074,64 +976,57 @@ class OperatorImplTest extends Specification { .groupTuple(size:2, remainder: true) then: - result.val == [1, ['a', 'b'] ] - result.val == [2, ['x', 'y'] ] - result.val == [3, ['p', 'q'] ] - result.val == [1, ['c'], ['d']] - result.val == Channel.STOP + result.unwrap() == [1, ['a', 'b'] ] + result.unwrap() == [2, ['x', 'y'] ] + result.unwrap() == [3, ['p', 'q'] ] + result.unwrap() == [1, ['c'], ['d']] + result.unwrap() == Channel.STOP } def testChannelIfEmpty() { - - def result - when: - result = Channel.of(1,2,3).ifEmpty(100) + def result = Channel.of(1,2,3).ifEmpty(100) then: - result.val == 1 - result.val == 2 - result.val == 3 - result.val == Channel.STOP + result.unwrap() == 1 + result.unwrap() == 2 + result.unwrap() == 3 + result.unwrap() == Channel.STOP when: result = Channel.empty().ifEmpty(100) then: - result.val == 100 - result.val == Channel.STOP + result.unwrap() == 100 + result.unwrap() == Channel.STOP when: result = Channel.empty().ifEmpty { 1+2 } then: - result.val == 3 - result.val == Channel.STOP + result.unwrap() == 3 + result.unwrap() == Channel.STOP when: result = Channel.value(1).ifEmpty(100) then: result instanceof DataflowVariable - result.val == 1 + result.unwrap() == 1 when: result = Channel.empty().ifEmpty(100) then: result instanceof DataflowQueue - result.val == 100 - + result.unwrap() == 100 } def 'should create a channel given a list'() { - when: def result = [10,20,30].channel() then: - result.val == 10 - result.val == 20 - result.val == 30 - result.val == Channel.STOP - + result.unwrap() == 10 + result.unwrap() == 20 + result.unwrap() == 30 + result.unwrap() == Channel.STOP } - def 'should assign a channel to new variable' () { given: def session = new Session() @@ -1142,40 +1037,36 @@ class OperatorImplTest extends Specification { .set { result } then: - session.binding.result.val == 12 - session.binding.result.val == 22 - session.binding.result.val == 32 - session.binding.result.val == Channel.STOP - + session.binding.result.unwrap() == 12 + session.binding.result.unwrap() == 22 + session.binding.result.unwrap() == 32 + session.binding.result.unwrap() == Channel.STOP } def 'should always the same value' () { - when: - def x = Channel.value('Hello') + def result = Channel.value('Hello') then: - x.val == 'Hello' - x.val == 'Hello' - x.val == 'Hello' + result.unwrap() == 'Hello' + result.val == 'Hello' + result.val == 'Hello' } def 'should emit channel items until the condition is verified' () { - when: def result = Channel.of(1,2,3,4).until { it == 3 } then: - result.val == 1 - result.val == 2 - result.val == Channel.STOP + result.unwrap() == 1 + result.unwrap() == 2 + result.unwrap() == Channel.STOP when: result = Channel.of(1,2,3).until { it == 5 } then: - result.val == 1 - result.val == 2 - result.val == 3 - result.val == Channel.STOP - + result.unwrap() == 1 + result.unwrap() == 2 + result.unwrap() == 3 + result.unwrap() == Channel.STOP } @@ -1187,10 +1078,9 @@ class OperatorImplTest extends Specification { Channel.value('Hello').set { result } then: - session.binding.result.val == 'Hello' - session.binding.result.val == 'Hello' - session.binding.result.val == 'Hello' - + session.binding.result.unwrap() == 'Hello' + session.binding.result.unwrap() == 'Hello' + session.binding.result.unwrap() == 'Hello' } def 'should assign queue channel to a new variable' () { @@ -1201,10 +1091,10 @@ class OperatorImplTest extends Specification { Channel.of(1,2,3).set { result } then: - session.binding.result.val == 1 - session.binding.result.val == 2 - session.binding.result.val == 3 - session.binding.result.val == Channel.STOP + session.binding.result.unwrap() == 1 + session.binding.result.unwrap() == 2 + session.binding.result.unwrap() == 3 + session.binding.result.unwrap() == Channel.STOP } } diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/RandomSampleTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/RandomSampleTest.groovy index 9ce69c5e91..6212d2cb24 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/RandomSampleTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/RandomSampleTest.groovy @@ -37,7 +37,7 @@ class RandomSampleTest extends Specification { def sampler = new RandomSampleOp(ch, 10) when: - def result = (List)sampler.apply().toList().val + def result = (List)sampler.apply().toList().unwrap() then: result.size() == 10 result.unique().size() == 10 @@ -52,7 +52,7 @@ class RandomSampleTest extends Specification { def sampler = new RandomSampleOp(ch, 20) when: - def result = (List)sampler.apply().toList().val + def result = (List)sampler.apply().toList().unwrap() then: result.size() == 10 result.unique().size() == 10 @@ -66,7 +66,7 @@ class RandomSampleTest extends Specification { def sampler = new RandomSampleOp(ch, 10) when: - def result = (List)sampler.apply().toList().val + def result = (List)sampler.apply().toList().unwrap() then: result.size() == 10 result.unique().size() == 10 @@ -83,8 +83,8 @@ class RandomSampleTest extends Specification { def secondSampler = new RandomSampleOp(ch2, 10, seed) when: - def resultFirstRun = (List)firstSampler.apply().toList().val - def resultSecondRun = (List)secondSampler.apply().toList().val + def resultFirstRun = (List)firstSampler.apply().toList().unwrap() + def resultSecondRun = (List)secondSampler.apply().toList().unwrap() then: resultFirstRun == resultSecondRun diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/SetOpTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/SetOpTest.groovy index 3620277fb6..5accad0636 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/SetOpTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/SetOpTest.groovy @@ -33,9 +33,9 @@ class SetOpTest extends Dsl2Spec { foo | map { it *2 } /) then: - result.val == 2 - result.val == 4 - result.val == 6 + result.unwrap() == 2 + result.unwrap() == 4 + result.unwrap() == 6 when: result = dsl_eval(/ @@ -43,7 +43,7 @@ class SetOpTest extends Dsl2Spec { foo | map { it *2 } /) then: - result.val == 10 + result.unwrap() == 10 } def 'should invoke set with dot notation' () { @@ -53,9 +53,9 @@ class SetOpTest extends Dsl2Spec { foo.map { it *2 } /) then: - result.val == 2 - result.val == 4 - result.val == 6 + result.unwrap() == 2 + result.unwrap() == 4 + result.unwrap() == 6 when: result = dsl_eval(/ @@ -63,7 +63,7 @@ class SetOpTest extends Dsl2Spec { foo.map { it.toUpperCase() } /) then: - result.val == 'HELLO' + result.unwrap() == 'HELLO' } @@ -77,7 +77,7 @@ class SetOpTest extends Dsl2Spec { return foo /) then: - result.val == 'X' + result.unwrap() == 'X' when: result = dsl_eval(/ @@ -88,8 +88,8 @@ class SetOpTest extends Dsl2Spec { return bar /) then: - result[0].val == 'X' - result[1].val == 'Y' + result[0].unwrap() == 'X' + result[1].unwrap() == 'Y' } } diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/SplitFastaOperatorTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/SplitFastaOperatorTest.groovy index 74612ff488..65f0f38c6e 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/SplitFastaOperatorTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/SplitFastaOperatorTest.groovy @@ -87,11 +87,11 @@ class SplitFastaOperatorTest extends Specification { given: def records = Channel.of(fasta1, fasta2).splitFasta(record:[id:true]) expect: - records.val == [id:'1aboA'] - records.val == [id:'1ycsB'] - records.val == [id:'1pht'] - records.val == [id:'alpha123'] - records.val == Channel.STOP + records.unwrap() == [id:'1aboA'] + records.unwrap() == [id:'1ycsB'] + records.unwrap() == [id:'1pht'] + records.unwrap() == [id:'alpha123'] + records.unwrap() == Channel.STOP } @@ -104,11 +104,11 @@ class SplitFastaOperatorTest extends Specification { .map{ record, code -> [record.id, code] } expect: - result.val == ['1aboA', 'one'] - result.val == ['1ycsB', 'one'] - result.val == ['1pht', 'one'] - result.val == ['alpha123', 'two'] - result.val == Channel.STOP + result.unwrap() == ['1aboA', 'one'] + result.unwrap() == ['1ycsB', 'one'] + result.unwrap() == ['1pht', 'one'] + result.unwrap() == ['alpha123', 'two'] + result.unwrap() == Channel.STOP } @@ -119,11 +119,11 @@ class SplitFastaOperatorTest extends Specification { Channel.of(fasta1,fasta2).splitFasta(record:[id:true], into: target) expect: - target.val == [id:'1aboA'] - target.val == [id:'1ycsB'] - target.val == [id:'1pht'] - target.val == [id:'alpha123'] - target.val == Channel.STOP + target.unwrap() == [id:'1aboA'] + target.unwrap() == [id:'1ycsB'] + target.unwrap() == [id:'1pht'] + target.unwrap() == [id:'alpha123'] + target.unwrap() == Channel.STOP } @@ -154,10 +154,10 @@ class SplitFastaOperatorTest extends Specification { when: Channel.of(F1,F3).splitFasta(by:2, into: target) then: - target.val == '>1\nAAA\n>2\nBBB\n' - target.val == '>3\nCCC\n' - target.val == '>1\nEEE\n>2\nFFF\n' - target.val == '>3\nGGG\n' + target.unwrap() == '>1\nAAA\n>2\nBBB\n' + target.unwrap() == '>3\nCCC\n' + target.unwrap() == '>1\nEEE\n>2\nFFF\n' + target.unwrap() == '>3\nGGG\n' } def 'should apply count on multiple entries with a limit'() { @@ -195,9 +195,9 @@ class SplitFastaOperatorTest extends Specification { when: Channel.of(F1,F3).splitFasta(by:2, limit:4, into: target) then: - target.val == '>1\nAAA\n>2\nBBB\n' - target.val == '>3\nCCC\n>4\nDDD\n' - target.val == '>1\nEEE\n>2\nFFF\n' - target.val == '>3\nGGG\n>4\nHHH\n' + target.unwrap() == '>1\nAAA\n>2\nBBB\n' + target.unwrap() == '>3\nCCC\n>4\nDDD\n' + target.unwrap() == '>1\nEEE\n>2\nFFF\n' + target.unwrap() == '>3\nGGG\n>4\nHHH\n' } } diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/SplitFastqOperatorTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/SplitFastqOperatorTest.groovy index 6046f85308..aaca44fe80 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/SplitFastqOperatorTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/SplitFastqOperatorTest.groovy @@ -79,7 +79,7 @@ class SplitFastqOperatorTest extends Specification { when: def target = Channel.of(READS).splitFastq(by:2) then: - target.val == ''' + target.unwrap() == ''' @SRR636272.19519409/1 GGCCCGGCAGCAGGATGATGCTCTCCCGGGCCAAGCCGGCTGTGGGGAGCACCCCGCCGCAGGGGGACAGGCGGAGGAAGAAAGGGAAGAAGGTGCCACAGATCG + @@ -91,7 +91,7 @@ class SplitFastqOperatorTest extends Specification { ''' .stripIndent().leftTrim() - target.val == ''' + target.unwrap() == ''' @SRR636272.21107783/1 CGGGGAGCGCGGGCCCGGCAGCAGGATGATGCTCTCCCGGGCCAAGCCGGCTGTAGGGAGCACCCCGCCGCAGGGGGACAGGCGAGATCGGAAGAGCACACGTCT + @@ -103,7 +103,7 @@ class SplitFastqOperatorTest extends Specification { ''' .stripIndent().leftTrim() - target.val == Channel.STOP + target.unwrap() == Channel.STOP } def 'should split a fastq to gzip chunks' () { @@ -114,7 +114,7 @@ class SplitFastqOperatorTest extends Specification { when: def target = Channel.of(READS).splitFastq(by:2, compress:true, file:folder) then: - gunzip(target.val) == ''' + gunzip(target.unwrap()) == ''' @SRR636272.19519409/1 GGCCCGGCAGCAGGATGATGCTCTCCCGGGCCAAGCCGGCTGTGGGGAGCACCCCGCCGCAGGGGGACAGGCGGAGGAAGAAAGGGAAGAAGGTGCCACAGATCG + @@ -126,7 +126,7 @@ class SplitFastqOperatorTest extends Specification { ''' .stripIndent().leftTrim() - gunzip(target.val) == ''' + gunzip(target.unwrap()) == ''' @SRR636272.21107783/1 CGGGGAGCGCGGGCCCGGCAGCAGGATGATGCTCTCCCGGGCCAAGCCGGCTGTAGGGAGCACCCCGCCGCAGGGGGACAGGCGAGATCGGAAGAGCACACGTCT + @@ -138,7 +138,7 @@ class SplitFastqOperatorTest extends Specification { ''' .stripIndent().leftTrim() - target.val == Channel.STOP + target.unwrap() == Channel.STOP cleanup: folder.deleteDir() @@ -147,7 +147,7 @@ class SplitFastqOperatorTest extends Specification { def 'should split read pairs' () { when: - def result = Channel.of(['sample_id',READS,READS2]).splitFastq(by:1, elem:[1,2]).toList().val + def result = Channel.of(['sample_id',READS,READS2]).splitFastq(by:1, elem:[1,2]).toList().unwrap() then: result.size() ==4 @@ -218,7 +218,7 @@ class SplitFastqOperatorTest extends Specification { when: channel = Channel.of(['sample_id',file1,file2]).splitFastq(by:1, pe:true) - result = channel.val + result = channel.unwrap() then: result[0] == 'sample_id' result[1] == ''' @@ -235,7 +235,7 @@ class SplitFastqOperatorTest extends Specification { '''.stripIndent().leftTrim() when: - result = channel.val + result = channel.unwrap() then: result[0] == 'sample_id' result[1] == ''' @@ -252,7 +252,7 @@ class SplitFastqOperatorTest extends Specification { '''.stripIndent().leftTrim() when: - result = channel.val + result = channel.unwrap() then: result[0] == 'sample_id' result[1] == ''' @@ -269,7 +269,7 @@ class SplitFastqOperatorTest extends Specification { '''.stripIndent().leftTrim() when: - result = channel.val + result = channel.unwrap() then: result[0] == 'sample_id' result[1] == ''' @@ -286,7 +286,7 @@ class SplitFastqOperatorTest extends Specification { '''.stripIndent().leftTrim() when: - result = channel.val + result = channel.unwrap() then: result == Channel.STOP @@ -307,7 +307,7 @@ class SplitFastqOperatorTest extends Specification { channel = Channel .from([ ['aaa_id',file_a_1,file_a_2], ['bbb_id',file_b_1,file_b_2] ]) .splitFastq(by:1, pe:true, file:folder) - result = channel.val + result = channel.unwrap() then: result[0] == 'aaa_id' result[1].name == 'aaa_1.1.fq' @@ -326,7 +326,7 @@ class SplitFastqOperatorTest extends Specification { '''.stripIndent().leftTrim() when: - result = channel.val + result = channel.unwrap() then: result[0] == 'aaa_id' result[1].name == 'aaa_1.2.fq' @@ -345,7 +345,7 @@ class SplitFastqOperatorTest extends Specification { '''.stripIndent().leftTrim() when: - result = channel.val + result = channel.unwrap() then: result[0] == 'aaa_id' result[1].name == 'aaa_1.3.fq' @@ -364,7 +364,7 @@ class SplitFastqOperatorTest extends Specification { '''.stripIndent().leftTrim() when: - result = channel.val + result = channel.unwrap() then: result[0] == 'aaa_id' result[1].name == 'aaa_1.4.fq' @@ -383,7 +383,7 @@ class SplitFastqOperatorTest extends Specification { '''.stripIndent().leftTrim() when: - result = channel.val + result = channel.unwrap() then: result[0] == 'bbb_id' result[1].name == 'bbb_1.1.fq' @@ -402,28 +402,28 @@ class SplitFastqOperatorTest extends Specification { '''.stripIndent().leftTrim() when: - result = channel.val + result = channel.unwrap() then: result[0] == 'bbb_id' result[1].name == 'bbb_1.2.fq' result[2].name == 'bbb_2.2.fq' when: - result = channel.val + result = channel.unwrap() then: result[0] == 'bbb_id' result[1].name == 'bbb_1.3.fq' result[2].name == 'bbb_2.3.fq' when: - result = channel.val + result = channel.unwrap() then: result[0] == 'bbb_id' result[1].name == 'bbb_1.4.fq' result[2].name == 'bbb_2.4.fq' when: - result = channel.val + result = channel.unwrap() then: result == Channel.STOP sleep 1_000 diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/ViewOperatorTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/ViewOperatorTest.groovy index 12f6c6511b..82a564027d 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/ViewOperatorTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/ViewOperatorTest.groovy @@ -44,10 +44,10 @@ class ViewOperatorTest extends Specification{ when: def result = Channel.of(1,2,3).view() then: - result.val == 1 - result.val == 2 - result.val == 3 - result.val == Channel.STOP + result.unwrap() == 1 + result.unwrap() == 2 + result.unwrap() == 3 + result.unwrap() == Channel.STOP capture.toString() == '1\n2\n3\n' } @@ -57,10 +57,10 @@ class ViewOperatorTest extends Specification{ when: def result = Channel.of(1,2,3).view { "~ $it " } then: - result.val == 1 - result.val == 2 - result.val == 3 - result.val == Channel.STOP + result.unwrap() == 1 + result.unwrap() == 2 + result.unwrap() == 3 + result.unwrap() == Channel.STOP capture.toString() == '~ 1 \n~ 2 \n~ 3 \n' @@ -71,10 +71,10 @@ class ViewOperatorTest extends Specification{ when: def result = Channel.of(1,2,3).view(newLine:false) { " ~ $it" } then: - result.val == 1 - result.val == 2 - result.val == 3 - result.val == Channel.STOP + result.unwrap() == 1 + result.unwrap() == 2 + result.unwrap() == 3 + result.unwrap() == Channel.STOP capture.toString() == ' ~ 1 ~ 2 ~ 3' } @@ -83,7 +83,7 @@ class ViewOperatorTest extends Specification{ when: def result = Channel.value(1).view { ">> $it" } then: - result.val == 1 + result.unwrap() == 1 capture.toString() == ">> 1\n" } @@ -91,7 +91,7 @@ class ViewOperatorTest extends Specification{ when: def result = Channel.value(Channel.STOP).view { ">> $it" } then: - result.val == Channel.STOP + result.unwrap() == Channel.STOP capture.toString() == '' } diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/plugin/ChannelFactoryInstanceTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/plugin/ChannelFactoryInstanceTest.groovy index ea2fb42193..a040a8b188 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/plugin/ChannelFactoryInstanceTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/plugin/ChannelFactoryInstanceTest.groovy @@ -98,10 +98,10 @@ class ChannelFactoryInstanceTest extends Specification { def runner = new MockScriptRunner() def result = runner.setScript(SCRIPT).execute() then: - result.val == 'one' - result.val == 'two' - result.val == 'three' - result.val == Channel.STOP + result.unwrap() == 'one' + result.unwrap() == 'two' + result.unwrap() == 'three' + result.unwrap() == Channel.STOP and: ext1.initCount == 1 ext1.initSession instanceof Session @@ -128,10 +128,10 @@ class ChannelFactoryInstanceTest extends Specification { def runner = new MockScriptRunner() def result = runner.setScript(SCRIPT).execute() then: - result.val == 'one' - result.val == 'two' - result.val == 'three' - result.val == Channel.STOP + result.unwrap() == 'one' + result.unwrap() == 'two' + result.unwrap() == 'three' + result.unwrap() == Channel.STOP and: ext1.initCount == 1 ext1.initSession instanceof Session @@ -178,7 +178,7 @@ class ChannelFactoryInstanceTest extends Specification { def runner = new MockScriptRunner() def result = runner.setScript(SCRIPT).execute() then: - result.val == ['1 X', '2 Y', '3 Z'] + result.unwrap() == ['1 X', '2 Y', '3 Z'] and: ext1.initCount == 1 @@ -209,9 +209,9 @@ class ChannelFactoryInstanceTest extends Specification { def runner = new MockScriptRunner() def result = runner.setScript(SCRIPT).execute() then: - result.val == 2 - result.val == 3 - result.val == 4 + result.unwrap() == 2 + result.unwrap() == 3 + result.unwrap() == 4 and: ext1.initCount == 1 diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/plugin/PluginExtensionProviderTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/plugin/PluginExtensionProviderTest.groovy index a32ebdc287..54a71530b8 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/plugin/PluginExtensionProviderTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/plugin/PluginExtensionProviderTest.groovy @@ -55,8 +55,8 @@ class PluginExtensionProviderTest extends Specification { def result = ext.invokeExtensionMethod(ch, 'map', { it -> it * it }) then: result instanceof DataflowReadChannel - result.val == 1 - result.val == 4 - result.val == 9 + result.unwrap() == 1 + result.unwrap() == 4 + result.unwrap() == 9 } } From 1f49d9261714edc334a911145e5e1e4c0dd3b987 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 7 Jan 2025 23:07:48 +0800 Subject: [PATCH 07/66] Fix tests Signed-off-by: Paolo Di Tommaso --- .../main/groovy/nextflow/extension/Op.groovy | 14 ++---- .../nextflow/extension/OperatorImpl.groovy | 3 +- .../nextflow/extension/UntilManyOp.groovy | 2 +- .../nextflow/extension/UntilManyOpTest.groovy | 50 +++++++++---------- 4 files changed, 32 insertions(+), 37 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/Op.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/Op.groovy index 7e63ea3f0b..5f4a997725 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/Op.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/Op.groovy @@ -17,7 +17,7 @@ package nextflow.extension -import groovy.transform.CompileDynamic + import groovy.transform.CompileStatic import groovy.transform.PackageScope import groovy.util.logging.Slf4j @@ -28,6 +28,7 @@ import nextflow.Session import nextflow.prov.OperatorRun import nextflow.prov.Prov import nextflow.prov.Tracker +import org.codehaus.groovy.runtime.InvokerHelper /** * Operator helpers methods @@ -131,7 +132,6 @@ class Op { } @Override - @CompileDynamic Object call(final Object... args) { // when the accumulator flag true, re-use the previous run object final run = !accumulator || previousRun==null @@ -141,9 +141,7 @@ class Op { currentOperator.set(run) // map the inputs final inputs = Prov.getTracker().receiveInputs(run, args.toList()) - final arr = inputs.toArray() - // todo: the spread operator should be replaced with proper array - final ret = target.call(*arr) + final ret = InvokerHelper.invokeMethod(target, 'call', inputs.toArray()) // track the previous run if( accumulator ) previousRun = run @@ -152,14 +150,12 @@ class Op { } Object call(Object args) { - // todo: this should invoke the above one - target.call(args) + call(InvokerHelper.asArray(args)) } @Override Object call() { - // todo: this should invoke the above one - target.call() + call(InvokerHelper.EMPTY_ARGS) } } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy index 4072acfcd4..e0163cd14d 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy @@ -479,8 +479,7 @@ class OperatorImpl { } DataflowWriteChannel collect(final DataflowReadChannel source, Map opts, Closure action=null) { - final target = new CollectOp(source,action,opts).apply() - return target + return new CollectOp(source,action,opts).apply() } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/UntilManyOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/UntilManyOp.groovy index 7ec5e49dd2..42f05d8ca6 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/UntilManyOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/UntilManyOp.groovy @@ -103,7 +103,7 @@ class UntilManyOp { @Override Object call(final Object arguments) { - throw new UnsupportedOperationException() + call(InvokerHelper.asArray(arguments)) } @Override diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/UntilManyOpTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/UntilManyOpTest.groovy index 706771b1b9..3625199211 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/UntilManyOpTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/UntilManyOpTest.groovy @@ -34,18 +34,18 @@ class UntilManyOpTest extends Specification { def source = Channel.of(1,2,3,4) def result = new UntilManyOp([source], { it==3 }).apply().get(0) then: - result.val == 1 - result.val == 2 - result.val == Channel.STOP + result.unwrap() == 1 + result.unwrap() == 2 + result.unwrap() == Channel.STOP when: source = Channel.of(1,2,3) result = new UntilManyOp([source], { it==5 }).apply().get(0) then: - result.val == 1 - result.val == 2 - result.val == 3 - result.val == Channel.STOP + result.unwrap() == 1 + result.unwrap() == 2 + result.unwrap() == 3 + result.unwrap() == Channel.STOP } @@ -60,17 +60,17 @@ class UntilManyOpTest extends Specification { def (X,Y,Z) = new UntilManyOp([A,B,C], condition).apply() then: - X.val == 1 - Y.val == 'alpha' - Z.val == 'foo' + X.unwrap() == 1 + Y.unwrap() == 'alpha' + Z.unwrap() == 'foo' and: - X.val == 2 - Y.val == 'beta' - Z.val == 'bar' + X.unwrap() == 2 + Y.unwrap() == 'beta' + Z.unwrap() == 'bar' and: - X.val == Channel.STOP - Y.val == Channel.STOP - Z.val == Channel.STOP + X.unwrap() == Channel.STOP + Y.unwrap() == Channel.STOP + Z.unwrap() == Channel.STOP } def 'should emit channels until list condition is verified' () { @@ -84,17 +84,17 @@ class UntilManyOpTest extends Specification { def (X,Y,Z) = new UntilManyOp([A,B,C], condition).apply() then: - X.val == 1 - Y.val == 'alpha' - Z.val == 'foo' + X.unwrap() == 1 + Y.unwrap() == 'alpha' + Z.unwrap() == 'foo' and: - X.val == 2 - Y.val == 'beta' - Z.val == 'bar' + X.unwrap() == 2 + Y.unwrap() == 'beta' + Z.unwrap() == 'bar' and: - X.val == Channel.STOP - Y.val == Channel.STOP - Z.val == Channel.STOP + X.unwrap() == Channel.STOP + Y.unwrap() == Channel.STOP + Z.unwrap() == Channel.STOP } } From 2ab3a6b472baa764166a378a7140d23bd3384753 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 7 Jan 2025 23:26:36 +0800 Subject: [PATCH 08/66] Fix tests Signed-off-by: Paolo Di Tommaso --- .../test/groovy/nextflow/ChannelTest.groovy | 296 +++++++++--------- .../nextflow/processor/TaskBeanTest.groovy | 2 +- .../nextflow/script/ScriptDslTest.groovy | 42 +-- .../nextflow/script/ScriptIncludesTest.groovy | 50 +-- .../nextflow/script/ScriptRecurseTest.groovy | 30 +- .../nextflow/script/ScriptRunnerTest.groovy | 24 +- .../script/params/EachInParamTest.groovy | 10 +- .../script/params/ParamsInTest.groovy | 110 +++---- 8 files changed, 282 insertions(+), 282 deletions(-) diff --git a/modules/nextflow/src/test/groovy/nextflow/ChannelTest.groovy b/modules/nextflow/src/test/groovy/nextflow/ChannelTest.groovy index c7b45a6395..90cee0b717 100644 --- a/modules/nextflow/src/test/groovy/nextflow/ChannelTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/ChannelTest.groovy @@ -47,59 +47,59 @@ class ChannelTest extends Specification { when: result = Channel.of('a') then: - result.val == 'a' - result.val == Channel.STOP + result.unwrap() == 'a' + result.unwrap() == Channel.STOP when: result = Channel.of('a','b','c') then: - result.val == 'a' - result.val == 'b' - result.val == 'c' - result.val == Channel.STOP + result.unwrap() == 'a' + result.unwrap() == 'b' + result.unwrap() == 'c' + result.unwrap() == Channel.STOP when: result = Channel.of([1,2,3]) then: - result.val == [1,2,3] - result.val == Channel.STOP + result.unwrap() == [1,2,3] + result.unwrap() == Channel.STOP when: result = Channel.of([1,2], [3,4]) then: - result.val == [1,2] - result.val == [3,4] - result.val == Channel.STOP + result.unwrap() == [1,2] + result.unwrap() == [3,4] + result.unwrap() == Channel.STOP when: result = Channel.of([]) then: - result.val == [] - result.val == Channel.STOP + result.unwrap() == [] + result.unwrap() == Channel.STOP when: result = Channel.of() then: - result.val == Channel.STOP + result.unwrap() == Channel.STOP when: result = Channel.of([1,2,3].toArray()) then: - result.val == 1 - result.val == 2 - result.val == 3 - result.val == Channel.STOP + result.unwrap() == 1 + result.unwrap() == 2 + result.unwrap() == 3 + result.unwrap() == Channel.STOP when: result = Channel.of([].toArray()) then: - result.val == Channel.STOP + result.unwrap() == Channel.STOP when: result = Channel.of(null) then: - result.val == null - result.val == Channel.STOP + result.unwrap() == null + result.unwrap() == Channel.STOP } @@ -111,30 +111,30 @@ class ChannelTest extends Specification { when: result = Channel.of(1..3) then: - result.val == 1 - result.val == 2 - result.val == 3 - result.val == Channel.STOP + result.unwrap() == 1 + result.unwrap() == 2 + result.unwrap() == 3 + result.unwrap() == Channel.STOP when: result = Channel.of(1..3,'X','Y') then: - result.val == 1 - result.val == 2 - result.val == 3 - result.val == 'X' - result.val == 'Y' - result.val == Channel.STOP + result.unwrap() == 1 + result.unwrap() == 2 + result.unwrap() == 3 + result.unwrap() == 'X' + result.unwrap() == 'Y' + result.unwrap() == Channel.STOP when: result = Channel.of(1..3,'X'..'Y') then: - result.val == 1 - result.val == 2 - result.val == 3 - result.val == 'X' - result.val == 'Y' - result.val == Channel.STOP + result.unwrap() == 1 + result.unwrap() == 2 + result.unwrap() == 3 + result.unwrap() == 'X' + result.unwrap() == 'Y' + result.unwrap() == Channel.STOP } def 'should create channel from a list'() { @@ -145,26 +145,26 @@ class ChannelTest extends Specification { when: result = Channel.fromList(['alpha','delta']) then: - result.val == 'alpha' - result.val == 'delta' - result.val == Channel.STOP + result.unwrap() == 'alpha' + result.unwrap() == 'delta' + result.unwrap() == Channel.STOP when: result = Channel.fromList([]) then: - result.val == Channel.STOP + result.unwrap() == Channel.STOP when: result = Channel.fromList(null) then: - result.val == Channel.STOP + result.unwrap() == Channel.STOP when: result = Channel.fromList([1..3, 'X'..'Y']) then: - result.val == 1..3 - result.val == 'X'..'Y' - result.val == Channel.STOP + result.unwrap() == 1..3 + result.unwrap() == 'X'..'Y' + result.unwrap() == Channel.STOP } def testFrom() { @@ -174,27 +174,27 @@ class ChannelTest extends Specification { when: result = Channel.from('hola') then: - result.val == 'hola' - result.val == Channel.STOP + result.unwrap() == 'hola' + result.unwrap() == Channel.STOP when: result = Channel.from('alpha','delta') then: - result.val == 'alpha' - result.val == 'delta' - result.val == Channel.STOP + result.unwrap() == 'alpha' + result.unwrap() == 'delta' + result.unwrap() == Channel.STOP when: result = Channel.from(['alpha','delta']) then: - result.val == 'alpha' - result.val == 'delta' - result.val == Channel.STOP + result.unwrap() == 'alpha' + result.unwrap() == 'delta' + result.unwrap() == Channel.STOP when: result = Channel.from([]) then: - result.val == Channel.STOP + result.unwrap() == Channel.STOP } def testSingleFile() { @@ -202,12 +202,12 @@ class ChannelTest extends Specification { when: def channel = Channel.fromPath('/some/file.txt') then: - channel.val == Paths.get('/some/file.txt') + channel.unwrap() == Paths.get('/some/file.txt') when: channel = Channel.fromPath('/some/f{i}le.txt') then: - channel.val == Paths.get('/some/f{i}le.txt') + channel.unwrap() == Paths.get('/some/f{i}le.txt') } @@ -228,42 +228,42 @@ class ChannelTest extends Specification { when: def result = Channel .fromPath("$folder/{alpha,gamma}.txt") - .toSortedList().getVal().collect { it -> it.name } + .toSortedList().unwrap().collect { it -> it.name } then: result == [ 'alpha.txt', 'gamma.txt' ] when: result = Channel .fromPath("$folder/file?.txt") - .toSortedList().getVal().collect { it -> it.name } + .toSortedList().unwrap().collect { it -> it.name } then: result == [ 'file4.txt', 'file5.txt' ] when: result = Channel .fromPath("$folder/file*.txt") - .toSortedList().getVal().collect { it -> it.name } + .toSortedList().unwrap().collect { it -> it.name } then: result == [ 'file4.txt', 'file5.txt', 'file66.txt' ] when: result = Channel .fromPath("$folder/{alpha,}.txt") - .toSortedList().getVal().collect { it -> it.name } + .toSortedList().unwrap().collect { it -> it.name } then: result == ['alpha.txt'] when: result = Channel .fromPath("$folder/{,beta}.txt") - .toSortedList().getVal().collect { it -> it.name } + .toSortedList().unwrap().collect { it -> it.name } then: result == ['beta.txt'] when: result = Channel .fromPath("$folder/alpha.txt{,}") - .toSortedList().getVal().collect { it -> it.name } + .toSortedList().unwrap().collect { it -> it.name } then: result == ['alpha.txt'] } @@ -276,21 +276,21 @@ class ChannelTest extends Specification { when: def result = Channel .fromPath([relative:false], 'alpha.txt') - .toSortedList().getVal() + .toSortedList().unwrap() then: result*.toString() == [ file1.absolutePath ] when: result = Channel .fromPath([relative:true], 'alpha.txt') - .toSortedList().getVal() + .toSortedList().unwrap() then: result*.toString() == [ 'alpha.txt' ] when: result = Channel .fromPath([:], 'alpha.txt') //no relative option set - .toSortedList().getVal() + .toSortedList().unwrap() then: result*.toString() == [ file1.absolutePath ] @@ -308,17 +308,17 @@ class ChannelTest extends Specification { def file4 = Files.createFile(folder.resolve('gamma.txt')) when: - def result = Channel.fromPath("$folder/*").toSortedList().getVal() + def result = Channel.fromPath("$folder/*").toSortedList().unwrap() then: result == [file3, file4] when: - result = Channel.fromPath("$folder/.*").toSortedList().getVal() + result = Channel.fromPath("$folder/.*").toSortedList().unwrap() then: result == [file1, file2] when: - result = Channel.fromPath("$folder/{.*,*}", hidden: true).toSortedList().getVal() + result = Channel.fromPath("$folder/{.*,*}", hidden: true).toSortedList().unwrap() then: result == [file1, file2, file3, file4] @@ -337,17 +337,17 @@ class ChannelTest extends Specification { def file6 = Files.createFile(sub1.resolve('file6.txt')) when: - def result = Channel.fromPath("$folder/*.txt").toSortedList().getVal() + def result = Channel.fromPath("$folder/*.txt").toSortedList().unwrap() then: result == [file1, file2, file3] when: - def result2 = Channel.fromPath("$folder/**.txt").toSortedList().getVal() + def result2 = Channel.fromPath("$folder/**.txt").toSortedList().unwrap() then: result2 == [file1, file2, file3, file6] when: - def result3 = Channel.fromPath("$folder/sub1/**.log").toSortedList().getVal() + def result3 = Channel.fromPath("$folder/sub1/**.log").toSortedList().unwrap() then: result3 == [file5] @@ -365,17 +365,17 @@ class ChannelTest extends Specification { def file6 = Files.createFile(sub1.resolve('file6.txt')) when: - def result = Channel.fromPath("$folder/file\\*.txt").toSortedList().getVal() + def result = Channel.fromPath("$folder/file\\*.txt").toSortedList().unwrap() then: result == [file2] when: - result = Channel.fromPath("$folder/file*.txt", glob: false).toSortedList().getVal() + result = Channel.fromPath("$folder/file*.txt", glob: false).toSortedList().unwrap() then: result == [file2] when: - result = Channel.fromPath("$folder/sub\\[a-b\\]/file*").toSortedList().getVal() + result = Channel.fromPath("$folder/sub\\[a-b\\]/file*").toSortedList().unwrap() then: result == [file5,file6] @@ -395,56 +395,56 @@ class ChannelTest extends Specification { when: List result = Channel .fromPath( folder.toAbsolutePath().toString() + '/*.txt' ) - .toSortedList().getVal().collect { it.name } + .toSortedList().unwrap().collect { it.name } then: result == [ 'file1.txt', 'file2.txt' ] when: result = Channel .fromPath( folder.toAbsolutePath().toString() + '/*' ) - .toSortedList().getVal().collect { it.name } + .toSortedList().unwrap().collect { it.name } then: result == [ 'file1.txt', 'file2.txt', 'file3.log' ] when: result = Channel .fromPath( folder.toAbsolutePath().toString() + '/*', type: 'file' ) - .toSortedList().getVal().collect { it.name } + .toSortedList().unwrap().collect { it.name } then: result == [ 'file1.txt', 'file2.txt', 'file3.log' ] when: result = Channel .fromPath( folder.toAbsolutePath().toString() + '/*', type: 'dir' ) - .toSortedList().getVal().collect { it.name } + .toSortedList().unwrap().collect { it.name } then: result == ['sub1'] when: result = Channel .fromPath( folder.toAbsolutePath().toString() + '/*', type: 'any' ) - .toSortedList().getVal().collect { it.name } + .toSortedList().unwrap().collect { it.name } then: result == [ 'file1.txt', 'file2.txt', 'file3.log', 'sub1' ] when: result = Channel .fromPath( folder.toAbsolutePath().toString() + '/**', type: 'file' ) - .toSortedList() .getVal() .collect { it.name } + .toSortedList() .unwrap() .collect { it.name } then: result == [ 'file1.txt', 'file2.txt', 'file3.log', 'file5.log' ] when: def result2 = Channel .fromPath( folder.toAbsolutePath().toString() + '/**', type: 'file', maxDepth: 0 ) - .toSortedList() .getVal() .collect { it.name } + .toSortedList() .unwrap() .collect { it.name } then: result2 == ['file1.txt', 'file2.txt', 'file3.log' ] when: def result3 = Channel .fromPath (folder.toAbsolutePath().toString() + '/{file1.txt,sub1/file5.log}') - .toSortedList() .val .collect { it.name } + .toSortedList() .unwrap() .collect { it.name } then: result3 == ['file1.txt','file5.log'] @@ -472,14 +472,14 @@ class ChannelTest extends Specification { when: List result = Channel .fromPath( '*.txt' ) - .toSortedList().getVal().collect { it.toString() } + .toSortedList().unwrap().collect { it.toString() } then: result == [ file1.toString(), file2.toString() ] when: result = Channel .fromPath( '*.txt', relative: true ) - .toSortedList().getVal().collect { it.toString() } + .toSortedList().unwrap().collect { it.toString() } then: result == [ file1.name, file2.name ] @@ -499,14 +499,14 @@ class ChannelTest extends Specification { when: List result = Channel .fromPath( 'file3.log' ) - .toSortedList().getVal().collect { it.toString() } + .toSortedList().unwrap().collect { it.toString() } then: result == [ file3.toString() ] when: result = Channel .fromPath( 'file3.log', relative: true ) - .toSortedList().getVal().collect { it.toString() } + .toSortedList().unwrap().collect { it.toString() } then: result == [ file3.name ] @@ -531,13 +531,13 @@ class ChannelTest extends Specification { // -- by default traverse symlinks when: - def result = Channel.fromPath( folder.toAbsolutePath().toString() + '/**/*.txt' ).toSortedList({it.name}).getVal().collect { it.getName() } + def result = Channel.fromPath( folder.toAbsolutePath().toString() + '/**/*.txt' ).toSortedList({it.name}).unwrap().collect { it.getName() } then: result == ['file3.txt','file3.txt','file4.txt','file4.txt'] // -- switch off symlinks traversing when: - def result2 = Channel.fromPath( folder.toAbsolutePath().toString() + '/**/*.txt', followLinks: false ).toSortedList({it.name}).getVal().collect { it.getName() } + def result2 = Channel.fromPath( folder.toAbsolutePath().toString() + '/**/*.txt', followLinks: false ).toSortedList({it.name}).unwrap().collect { it.getName() } then: result2 == ['file3.txt','file4.txt'] @@ -554,17 +554,17 @@ class ChannelTest extends Specification { // -- by default no hidden when: - def result = Channel.fromPath( folder.toAbsolutePath().toString() + '/*.txt' ).toSortedList({it.name}).getVal().collect { it.getName() } + def result = Channel.fromPath( folder.toAbsolutePath().toString() + '/*.txt' ).toSortedList({it.name}).unwrap().collect { it.getName() } then: result == ['file1.txt','file2.txt'] when: - result = Channel.fromPath( folder.toAbsolutePath().toString() + '/.*.txt' ).toSortedList({it.name}).getVal().collect { it.getName() } + result = Channel.fromPath( folder.toAbsolutePath().toString() + '/.*.txt' ).toSortedList({it.name}).unwrap().collect { it.getName() } then: result == ['.file_hidden.txt'] when: - result = Channel.fromPath( folder.toAbsolutePath().toString() + '/*.txt', hidden: true ).toSortedList({it.name}).getVal().collect { it.getName() } + result = Channel.fromPath( folder.toAbsolutePath().toString() + '/*.txt', hidden: true ).toSortedList({it.name}).unwrap().collect { it.getName() } then: result == ['.file_hidden.txt', 'file1.txt','file2.txt'] @@ -580,17 +580,17 @@ class ChannelTest extends Specification { folder.resolve('file3.fq').text = 'Ciao' when: - def result = Channel.fromPath( ["$folder/*.txt", "$folder/*.fq"] ).toSortedList({it.name}).getVal().collect { it.getName() } + def result = Channel.fromPath( ["$folder/*.txt", "$folder/*.fq"] ).toSortedList({it.name}).unwrap().collect { it.getName() } then: result == [ 'file1.txt', 'file2.txt', 'file3.fq' ] when: - result = Channel.fromPath( ["$folder/file1.txt", "$folder/file2.txt", "$folder/file3.fq"] ).toSortedList({it.name}).getVal().collect { it.getName() } + result = Channel.fromPath( ["$folder/file1.txt", "$folder/file2.txt", "$folder/file3.fq"] ).toSortedList({it.name}).unwrap().collect { it.getName() } then: result == [ 'file1.txt', 'file2.txt', 'file3.fq' ] when: - result = Channel.fromPath( ["$folder/*"] ).toSortedList({it.name}).getVal().collect { it.getName() } + result = Channel.fromPath( ["$folder/*"] ).toSortedList({it.name}).unwrap().collect { it.getName() } then: result == [ 'file1.txt', 'file2.txt', 'file3.fq' ] } @@ -611,21 +611,21 @@ class ChannelTest extends Specification { when: def ch = Channel.fromPath(file1.toString()) then: - ch.getVal() == file1 - ch.getVal() == Channel.STOP + ch.unwrap() == file1 + ch.unwrap() == Channel.STOP when: ch = Channel.fromPath([file1.toString(), file2.toString()]) then: - ch.getVal() == file1 - ch.getVal() == file2 - ch.getVal() == Channel.STOP + ch.unwrap() == file1 + ch.unwrap() == file2 + ch.unwrap() == Channel.STOP when: ch = Channel.fromPath(file1.toString(), checkIfExists: true) then: - ch.getVal() == file1 - ch.getVal() == Channel.STOP + ch.unwrap() == file1 + ch.unwrap() == Channel.STOP when: def session = new Session() @@ -680,7 +680,7 @@ class ChannelTest extends Specification { when: def result = Channel.fromPath("$folder/*.txt", checkIfExists: true) then: - result.getVal() instanceof Path + result.unwrap() instanceof Path !session.terminated when: @@ -743,18 +743,18 @@ class ChannelTest extends Specification { when: def pairs = Channel.fromFilePairs(folder.resolve("*_{1,2}.*")) then: - pairs.val == ['alpha', [a1, a2]] - pairs.val == ['beta', [b1, b2]] - pairs.val == ['delta', [d1, d2]] - pairs.val == Channel.STOP + pairs.unwrap() == ['alpha', [a1, a2]] + pairs.unwrap() == ['beta', [b1, b2]] + pairs.unwrap() == ['delta', [d1, d2]] + pairs.unwrap() == Channel.STOP when: pairs = Channel.fromFilePairs(folder.resolve("*_{1,2}.fa") , flat: true) then: - pairs.val == ['alpha', a1, a2] - pairs.val == ['beta', b1, b2] - pairs.val == ['delta', d1, d2] - pairs.val == Channel.STOP + pairs.unwrap() == ['alpha', a1, a2] + pairs.unwrap() == ['beta', b1, b2] + pairs.unwrap() == ['delta', d1, d2] + pairs.unwrap() == Channel.STOP } def 'should group files with the same prefix and root path' () { @@ -771,7 +771,7 @@ class ChannelTest extends Specification { SysEnv.push(NXF_FILE_ROOT: folder.toString()) when: - def pairs = Channel .fromFilePairs("*_{1,2}.*") .toList(). getVal() .sort { it[0] } + def pairs = Channel .fromFilePairs("*_{1,2}.*") .toList() .unwrap() .sort { it[0] } then: pairs == [ ['aa', [a1, a2]], @@ -797,9 +797,9 @@ class ChannelTest extends Specification { def grouping = { Path file -> file.name.substring(0,1) } def pairs = Channel.fromFilePairs(folder.resolve("*_{1,2}.*"), grouping, size:-1) then: - pairs.val == ['c', [c1, c2]] - pairs.val == ['h', [a1, a2, b1, b2]] - pairs.val == Channel.STOP + pairs.unwrap() == ['c', [c1, c2]] + pairs.unwrap() == ['h', [a1, a2, b1, b2]] + pairs.unwrap() == Channel.STOP } @@ -824,26 +824,26 @@ class ChannelTest extends Specification { // default size == 2 def pairs = Channel.fromFilePairs(folder.resolve("*_{1,2,3}.fa")) then: - pairs.val == ['alpha', [a1, a2]] - pairs.val == ['beta', [b1, b2]] - pairs.val == ['delta', [d1, d2]] - pairs.val == Channel.STOP + pairs.unwrap() == ['alpha', [a1, a2]] + pairs.unwrap() == ['beta', [b1, b2]] + pairs.unwrap() == ['delta', [d1, d2]] + pairs.unwrap() == Channel.STOP when: pairs = Channel.fromFilePairs(folder.resolve("*_{1,2,3,4}.fa"), size: 3) then: - pairs.val == ['alpha', [a1, a2, a3]] - pairs.val == ['beta', [b1, b2, b3]] - pairs.val == ['delta', [d1, d2, d3]] - pairs.val == Channel.STOP + pairs.unwrap() == ['alpha', [a1, a2, a3]] + pairs.unwrap() == ['beta', [b1, b2, b3]] + pairs.unwrap() == ['delta', [d1, d2, d3]] + pairs.unwrap() == Channel.STOP when: pairs = Channel.fromFilePairs(folder.resolve("*_{1,2,3,4}.fa"), size: -1) then: - pairs.val == ['alpha', [a1, a2, a3]] - pairs.val == ['beta', [b1, b2, b3]] - pairs.val == ['delta', [d1, d2, d3, d4]] - pairs.val == Channel.STOP + pairs.unwrap() == ['alpha', [a1, a2, a3]] + pairs.unwrap() == ['beta', [b1, b2, b3]] + pairs.unwrap() == ['delta', [d1, d2, d3, d4]] + pairs.unwrap() == Channel.STOP } @@ -864,11 +864,11 @@ class ChannelTest extends Specification { when: def pairs = Channel.fromFilePairs([folder.resolve("*_{1,2}.fa"), folder.resolve("$folder/*_{1,2}.fq")]) then: - pairs.val == ['alpha', [a1, a2]] - pairs.val == ['beta', [b1, b2]] - pairs.val == ['delta', [d1, d2]] - pairs.val == ['gamma', [g1, g2]] - pairs.val == Channel.STOP + pairs.unwrap() == ['alpha', [a1, a2]] + pairs.unwrap() == ['beta', [b1, b2]] + pairs.unwrap() == ['delta', [d1, d2]] + pairs.unwrap() == ['gamma', [g1, g2]] + pairs.unwrap() == Channel.STOP } @@ -881,9 +881,9 @@ class ChannelTest extends Specification { when: def files = Channel.fromFilePairs(folder.resolve('*.fa'), size:1) then: - files.val == ['alpha_1', [a1]] - files.val == ['alpha_2', [a2]] - files.val == Channel.STOP + files.unwrap() == ['alpha_1', [a1]] + files.unwrap() == ['alpha_2', [a2]] + files.unwrap() == Channel.STOP } def 'should return singleton' () { @@ -894,8 +894,8 @@ class ChannelTest extends Specification { when: def files = Channel.fromFilePairs(a1, size:1) then: - files.val == ['alpha_1', [a1]] - files.val == Channel.STOP + files.unwrap() == ['alpha_1', [a1]] + files.unwrap() == Channel.STOP } def 'should use size one by default' () { @@ -906,8 +906,8 @@ class ChannelTest extends Specification { when: def files = Channel.fromFilePairs(a1) then: - files.val == ['alpha_1', [a1]] - files.val == Channel.STOP + files.unwrap() == ['alpha_1', [a1]] + files.unwrap() == Channel.STOP } def 'should watch and emit a file' () { @@ -919,14 +919,14 @@ class ChannelTest extends Specification { sleep 500 Files.createFile(folder.resolve('hello.txt')) then: - result.val == folder.resolve('hello.txt') + result.unwrap() == folder.resolve('hello.txt') when: result = Channel.watchPath(folder.toString()) sleep 500 Files.createFile(folder.resolve('ciao.txt')) then: - result.val == folder.resolve('ciao.txt') + result.unwrap() == folder.resolve('ciao.txt') cleanup: folder?.deleteDir() @@ -938,7 +938,7 @@ class ChannelTest extends Specification { when: def result = Channel.watchPath("$folder/foo/*") then: - result.val == Channel.STOP + result.unwrap() == Channel.STOP cleanup: folder?.deleteDir() } @@ -954,10 +954,10 @@ class ChannelTest extends Specification { when: def result = Channel.fromFilePairs(files) then: - result.val == ['SRR389222_sub1', [Paths.get('/data/SRR389222_sub1.fastq.gz')]] - result.val == ['SRR389222_sub2', [Paths.get('/data/SRR389222_sub2.fastq.gz')]] - result.val == ['SRR389222_sub3', [Paths.get('/data/SRR389222_sub3.fastq.gz')]] - result.val == Channel.STOP + result.unwrap() == ['SRR389222_sub1', [Paths.get('/data/SRR389222_sub1.fastq.gz')]] + result.unwrap() == ['SRR389222_sub2', [Paths.get('/data/SRR389222_sub2.fastq.gz')]] + result.unwrap() == ['SRR389222_sub3', [Paths.get('/data/SRR389222_sub3.fastq.gz')]] + result.unwrap() == Channel.STOP } def 'should check file pair exists' () { @@ -984,9 +984,9 @@ class ChannelTest extends Specification { when: def result = Channel.fromFilePairs([ftp1, ftp2]) then: - result.val == ['SRR389222_sub1', [ftp1]] - result.val == ['SRR389222_sub2', [ftp2]] - result.val == Channel.STOP + result.unwrap() == ['SRR389222_sub1', [ftp1]] + result.unwrap() == ['SRR389222_sub2', [ftp2]] + result.unwrap() == Channel.STOP } diff --git a/modules/nextflow/src/test/groovy/nextflow/processor/TaskBeanTest.groovy b/modules/nextflow/src/test/groovy/nextflow/processor/TaskBeanTest.groovy index 4cfcb2a1fd..f2e16a0616 100644 --- a/modules/nextflow/src/test/groovy/nextflow/processor/TaskBeanTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/processor/TaskBeanTest.groovy @@ -54,7 +54,7 @@ class TaskBeanTest extends Specification { config.stageOutMode = 'rsync' def task = Mock(TaskRun) - task.getId() >> '123' + task.getId() >> TaskId.of('123') task.getName() >> 'Hello' task.getStdin() >> 'input from stdin' task.getScratch() >> '/tmp/x' diff --git a/modules/nextflow/src/test/groovy/nextflow/script/ScriptDslTest.groovy b/modules/nextflow/src/test/groovy/nextflow/script/ScriptDslTest.groovy index 3758ad497a..138a256147 100644 --- a/modules/nextflow/src/test/groovy/nextflow/script/ScriptDslTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/script/ScriptDslTest.groovy @@ -36,8 +36,8 @@ class ScriptDslTest extends Dsl2Spec { def runner = new MockScriptRunner() def result = runner.setScript(SCRIPT).execute() then: - result[0].val == 'Hello' - result[1].val == 'world' + result[0].unwrap() == 'Hello' + result[1].unwrap() == 'world' } def 'should execute basic workflow' () { @@ -52,7 +52,7 @@ class ScriptDslTest extends Dsl2Spec { ''' then: - result.val == 'Hello world' + result.unwrap() == 'Hello world' } def 'should execute emit' () { @@ -66,7 +66,7 @@ class ScriptDslTest extends Dsl2Spec { ''' then: - result.val == 'Hello world' + result.unwrap() == 'Hello world' } def 'should emit expression' () { @@ -82,7 +82,7 @@ class ScriptDslTest extends Dsl2Spec { ''' then: - result.val == 'HELLO WORLD' + result.unwrap() == 'HELLO WORLD' } def 'should emit process out' () { @@ -101,7 +101,7 @@ class ScriptDslTest extends Dsl2Spec { ''' then: - result.val == 'Hello' + result.unwrap() == 'Hello' } @@ -142,7 +142,7 @@ class ScriptDslTest extends Dsl2Spec { ''' then: - result.val == 'HELLO MUNDO' + result.unwrap() == 'HELLO MUNDO' } @@ -229,7 +229,7 @@ class ScriptDslTest extends Dsl2Spec { } /) then: - result.val == 'HELLO' + result.unwrap() == 'HELLO' } def 'should branch and view' () { @@ -250,12 +250,12 @@ class ScriptDslTest extends Dsl2Spec { [ch1, ch2] /) then: - result[0].val == 1 - result[0].val == 2 - result[0].val == 3 + result[0].unwrap() == 1 + result[0].unwrap() == 2 + result[0].unwrap() == 3 and: - result[1].val == 40 - result[1].val == 50 + result[1].unwrap() == 40 + result[1].unwrap() == 50 } @@ -278,9 +278,9 @@ class ScriptDslTest extends Dsl2Spec { ''') then: - result.val == 'hello' - result.val == 'world' - result.val == Channel.STOP + result.unwrap() == 'hello' + result.unwrap() == 'world' + result.unwrap() == Channel.STOP } def 'should allow process and operator composition' () { @@ -304,9 +304,9 @@ class ScriptDslTest extends Dsl2Spec { then: - result.val == 'hello' - result.val == 'world' - result.val == Channel.STOP + result.unwrap() == 'hello' + result.unwrap() == 'world' + result.unwrap() == Channel.STOP } def 'should run entry flow' () { @@ -335,7 +335,7 @@ class ScriptDslTest extends Dsl2Spec { then: - result.val == 'world' + result.unwrap() == 'world' } @@ -601,7 +601,7 @@ class ScriptDslTest extends Dsl2Spec { ''' then: - result.val == 'Hello' + result.unwrap() == 'Hello' } diff --git a/modules/nextflow/src/test/groovy/nextflow/script/ScriptIncludesTest.groovy b/modules/nextflow/src/test/groovy/nextflow/script/ScriptIncludesTest.groovy index 81a53a492f..54634490a7 100644 --- a/modules/nextflow/src/test/groovy/nextflow/script/ScriptIncludesTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/script/ScriptIncludesTest.groovy @@ -145,7 +145,7 @@ class ScriptIncludesTest extends Dsl2Spec { def result = runner.setScript(SCRIPT).execute() then: - result.val == 'dlrow olleh' + result.unwrap() == 'dlrow olleh' } def 'should allow duplicate functions' () { @@ -174,7 +174,7 @@ class ScriptIncludesTest extends Dsl2Spec { when: def result = new MockScriptRunner() .setScript(SCRIPT).execute() - def map = result.val + def map = result.unwrap() then: map map.witharg == 'hello world'.reverse() @@ -213,9 +213,9 @@ class ScriptIncludesTest extends Dsl2Spec { def result = new MockScriptRunner() .setScript(SCRIPT).execute() then: - result.val == '1-2' - result.val == '2-4' - result.val == '3-6' + result.unwrap() == '1-2' + result.unwrap() == '2-4' + result.unwrap() == '3-6' cleanup: NextflowMeta.instance.strictMode(false) @@ -301,7 +301,7 @@ class ScriptIncludesTest extends Dsl2Spec { def result = runner.setScript(SCRIPT).execute() then: - result.val == 'HELLO MUNDO' + result.unwrap() == 'HELLO MUNDO' binding.variables.alpha == null } @@ -351,7 +351,7 @@ class ScriptIncludesTest extends Dsl2Spec { def result = runner.setScript(SCRIPT).execute() then: - result.val == 'HELLO MUNDO' + result.unwrap() == 'HELLO MUNDO' !binding.hasVariable('alpha') !binding.hasVariable('foo') !binding.hasVariable('bar') @@ -397,7 +397,7 @@ class ScriptIncludesTest extends Dsl2Spec { def result = runner.setScript(SCRIPT).execute() then: - result.val == 'HELLO MUNDO' + result.unwrap() == 'HELLO MUNDO' } def 'should gather process outputs' () { @@ -440,7 +440,7 @@ class ScriptIncludesTest extends Dsl2Spec { def result = runner.setScript(SCRIPT).execute() then: - result.val == 'HELLO MUNDO' + result.unwrap() == 'HELLO MUNDO' !vars.containsKey('data') !vars.containsKey('foo') !vars.containsKey('bar') @@ -476,7 +476,7 @@ class ScriptIncludesTest extends Dsl2Spec { def result = runner.setScript(SCRIPT).execute() then: noExceptionThrown() - result.val == 'echo Hello world' + result.unwrap() == 'echo Hello world' cleanup: folder?.deleteDir() @@ -516,7 +516,7 @@ class ScriptIncludesTest extends Dsl2Spec { def result = runner.setScript(SCRIPT).execute() then: noExceptionThrown() - result.val == 'echo sample=world pairId=x reads=/some/file' + result.unwrap() == 'echo sample=world pairId=x reads=/some/file' } @@ -563,7 +563,7 @@ class ScriptIncludesTest extends Dsl2Spec { when: def result = dsl_eval(SCRIPT) then: - result.val == 'echo Ciao world' + result.unwrap() == 'echo Ciao world' } def 'should use multiple assignment' () { @@ -610,8 +610,8 @@ class ScriptIncludesTest extends Dsl2Spec { when: def result = dsl_eval(SCRIPT) then: - result[0].val == 'Ciao' - result[1].val == 'world' + result[0].unwrap() == 'Ciao' + result[1].unwrap() == 'world' } @@ -648,7 +648,7 @@ class ScriptIncludesTest extends Dsl2Spec { def result = runner.setScript(SCRIPT).execute() then: noExceptionThrown() - result.val == 'echo Hello world' + result.unwrap() == 'echo Hello world' } @@ -716,7 +716,7 @@ class ScriptIncludesTest extends Dsl2Spec { def result = runner.setScript(SCRIPT).execute() then: noExceptionThrown() - result.val == 'echo Hola mundo' + result.unwrap() == 'echo Hola mundo' } def 'should not fail when invoking a process in a module' () { @@ -866,8 +866,8 @@ class ScriptIncludesTest extends Dsl2Spec { def result = runner.setScript(SCRIPT).execute() then: - result[0].val == 'HELLO' - result[1].val == 'WORLD' + result[0].unwrap() == 'HELLO' + result[1].unwrap() == 'WORLD' } @@ -919,8 +919,8 @@ class ScriptIncludesTest extends Dsl2Spec { """) then: - result[0].val == 'CMD CONSUMER 1' - result[1].val == 'CMD CONSUMER 2' + result[0].unwrap() == 'CMD CONSUMER 1' + result[1].unwrap() == 'CMD CONSUMER 2' } @@ -957,8 +957,8 @@ class ScriptIncludesTest extends Dsl2Spec { def result = runner.setScript(SCRIPT).execute() then: - result[0].val == 'HELLO' - result[1].val == 'WORLD' + result[0].unwrap() == 'HELLO' + result[1].unwrap() == 'WORLD' } def 'should inherit module params' () { @@ -993,7 +993,7 @@ class ScriptIncludesTest extends Dsl2Spec { def result = runner.setScript(SCRIPT).execute() then: - result.val == 'OWNER LAST' + result.unwrap() == 'OWNER LAST' } def 'should override module params' () { @@ -1028,7 +1028,7 @@ class ScriptIncludesTest extends Dsl2Spec { def result = runner.setScript(SCRIPT).execute() then: - result.val == 'AAA ZZZ' + result.unwrap() == 'AAA ZZZ' } def 'should extends module params' () { @@ -1065,7 +1065,7 @@ class ScriptIncludesTest extends Dsl2Spec { def result = runner.setScript(SCRIPT).execute() then: - result.val == 'ONE ZZZ' + result.unwrap() == 'ONE ZZZ' } def 'should declare moduleDir path' () { diff --git a/modules/nextflow/src/test/groovy/nextflow/script/ScriptRecurseTest.groovy b/modules/nextflow/src/test/groovy/nextflow/script/ScriptRecurseTest.groovy index 2723b2a051..9a1282d81a 100644 --- a/modules/nextflow/src/test/groovy/nextflow/script/ScriptRecurseTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/script/ScriptRecurseTest.groovy @@ -53,10 +53,10 @@ class ScriptRecurseTest extends Dsl2Spec { def runner = new MockScriptRunner() def result = runner.setScript(SCRIPT).execute() then: - result.val == 2 - result.val == 3 - result.val == 4 - result.val == Channel.STOP + result.unwrap() == 2 + result.unwrap() == 3 + result.unwrap() == 4 + result.unwrap() == Channel.STOP } def 'should recourse a process until a condition is verified' () { @@ -79,10 +79,10 @@ class ScriptRecurseTest extends Dsl2Spec { def runner = new MockScriptRunner() def result = runner.setScript(SCRIPT).execute() then: - result.val == 2 - result.val == 3 - result.val == 4 - result.val == Channel.STOP + result.unwrap() == 2 + result.unwrap() == 3 + result.unwrap() == 4 + result.unwrap() == Channel.STOP } @@ -121,10 +121,10 @@ class ScriptRecurseTest extends Dsl2Spec { def runner = new MockScriptRunner() def result = runner.setScript(SCRIPT).execute() then: - result.val == 4 - result.val == 25 - result.val == 676 - result.val == Channel.STOP + result.unwrap() == 4 + result.unwrap() == 25 + result.unwrap() == 676 + result.unwrap() == Channel.STOP } def 'should recurse with scan' () { @@ -151,9 +151,9 @@ class ScriptRecurseTest extends Dsl2Spec { def runner = new MockScriptRunner() def result = runner.setScript(SCRIPT).execute() then: - result.val == 11 // 10 +1 - result.val == 32 // 20 + 11 +1 - result.val == 74 // 30 + 11 + 32 +1 + result.unwrap() == 11 // 10 +1 + result.unwrap() == 32 // 20 + 11 +1 + result.unwrap() == 74 // 30 + 11 + 32 +1 } diff --git a/modules/nextflow/src/test/groovy/nextflow/script/ScriptRunnerTest.groovy b/modules/nextflow/src/test/groovy/nextflow/script/ScriptRunnerTest.groovy index 682337368d..900eb3171c 100644 --- a/modules/nextflow/src/test/groovy/nextflow/script/ScriptRunnerTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/script/ScriptRunnerTest.groovy @@ -66,7 +66,7 @@ class ScriptRunnerTest extends Dsl2Spec { // when no outputs are specified, the 'stdout' is the default output then: result instanceof DataflowVariable - result.val == "echo Hello world" + result.unwrap() == "echo Hello world" } @@ -130,7 +130,7 @@ class ScriptRunnerTest extends Dsl2Spec { def result = new MockScriptRunner().setScript(script).execute() then: - result.val == 'echo 1 - 3' + result.unwrap() == 'echo 1 - 3' } @@ -160,7 +160,7 @@ class ScriptRunnerTest extends Dsl2Spec { runner.execute() then: - runner.result.val == 'echo 1' + runner.result.unwrap() == 'echo 1' TaskProcessor.currentProcessor().name == 'simpleTask' } @@ -191,7 +191,7 @@ class ScriptRunnerTest extends Dsl2Spec { def runner = new MockScriptRunner().setScript(script) def result = runner.execute() then: - result.val == '1-2-3' + result.unwrap() == '1-2-3' } @@ -221,7 +221,7 @@ class ScriptRunnerTest extends Dsl2Spec { def runner = new MockScriptRunner().setScript(script) def result = runner.execute() then: - result.val == '1-2-3' + result.unwrap() == '1-2-3' } @@ -272,7 +272,7 @@ class ScriptRunnerTest extends Dsl2Spec { def config = [executor: 'nope', env: [HELLO: 'Hello world!']] expect: - new MockScriptRunner(config).setScript(script).execute().val == 'Hello world!' + new MockScriptRunner(config).setScript(script).execute().unwrap() == 'Hello world!' } @@ -300,7 +300,7 @@ class ScriptRunnerTest extends Dsl2Spec { def config = [executor: 'nope'] expect: - new MockScriptRunner(config).setScript(script).execute().val == 'cat filename' + new MockScriptRunner(config).setScript(script).execute().unwrap() == 'cat filename' } @@ -536,7 +536,7 @@ class ScriptRunnerTest extends Dsl2Spec { def result = new MockScriptRunner(session) .setScript(script) .execute() - .getVal() + .unwrap() .toString() .stripIndent() .trim() @@ -586,7 +586,7 @@ class ScriptRunnerTest extends Dsl2Spec { def result = new MockScriptRunner(config) .setScript(script) .execute() - .getVal() + .unwrap() .toString() .stripIndent() .trim() @@ -692,7 +692,7 @@ class ScriptRunnerTest extends Dsl2Spec { // when no outputs are specified, the 'stdout' is the default output then: result instanceof DataflowVariable - result.val == "echo foo" + result.unwrap() == "echo foo" } @@ -729,7 +729,7 @@ class ScriptRunnerTest extends Dsl2Spec { // when no outputs are specified, the 'stdout' is the default output then: result instanceof DataflowVariable - result.val == "echo foo" + result.unwrap() == "echo foo" } @@ -767,7 +767,7 @@ class ScriptRunnerTest extends Dsl2Spec { // when no outputs are specified, the 'stdout' is the default output then: result instanceof DataflowVariable - result.val == "echo foo" + result.unwrap() == "echo foo" } } diff --git a/modules/nextflow/src/test/groovy/nextflow/script/params/EachInParamTest.groovy b/modules/nextflow/src/test/groovy/nextflow/script/params/EachInParamTest.groovy index 8fde47710d..1a66bb4fda 100644 --- a/modules/nextflow/src/test/groovy/nextflow/script/params/EachInParamTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/script/params/EachInParamTest.groovy @@ -39,11 +39,11 @@ class EachInParamTest extends Specification { def each = new EachInParam(Mock(Binding), []) expect: - each.normalizeToVariable(1).val == [1] - each.normalizeToVariable([3,4,5]).val == [3,4,5] - each.normalizeToVariable(channel).val == [1,2,3,5] - each.normalizeToVariable(value).val == ['a'] - each.normalizeToVariable(list).val == [4,5,6] + each.normalizeToVariable(1).unwrap() == [1] + each.normalizeToVariable([3,4,5]).unwrap() == [3,4,5] + each.normalizeToVariable(channel).unwrap() == [1,2,3,5] + each.normalizeToVariable(value).unwrap() == ['a'] + each.normalizeToVariable(list).unwrap() == [4,5,6] } diff --git a/modules/nextflow/src/test/groovy/nextflow/script/params/ParamsInTest.groovy b/modules/nextflow/src/test/groovy/nextflow/script/params/ParamsInTest.groovy index 9f3e412d85..8bc325bb0e 100644 --- a/modules/nextflow/src/test/groovy/nextflow/script/params/ParamsInTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/script/params/ParamsInTest.groovy @@ -72,21 +72,21 @@ class ParamsInTest extends Dsl2Spec { in1.class == ValueInParam in1.name == 'x' - in1.inChannel.val == 'Hello' + in1.inChannel.unwrap() == 'Hello' in2.class == ValueInParam in2.name == 'x' - in2.inChannel.val == 'Hola' + in2.inChannel.unwrap() == 'Hola' in3.class == ValueInParam in3.name == 'x' - in3.inChannel.val == 'ciao' + in3.inChannel.unwrap() == 'ciao' in4.class == ValueInParam in4.name == 'x' - in4.inChannel.val == 1 - in4.inChannel.val == 2 - in4.inChannel.val == Channel.STOP + in4.inChannel.unwrap() == 1 + in4.inChannel.unwrap() == 2 + in4.inChannel.unwrap() == Channel.STOP } @@ -122,19 +122,19 @@ class ParamsInTest extends Dsl2Spec { then: in1.name == 'x' - in1.inChannel.val == 1 - in1.inChannel.val == 2 - in1.inChannel.val == Channel.STOP + in1.inChannel.unwrap() == 1 + in1.inChannel.unwrap() == 2 + in1.inChannel.unwrap() == Channel.STOP in2.name == 'y' - in2.inChannel.val == 'a' - in2.inChannel.val == 'b' - in2.inChannel.val == Channel.STOP + in2.inChannel.unwrap() == 'a' + in2.inChannel.unwrap() == 'b' + in2.inChannel.unwrap() == Channel.STOP in3.name == 'z' - in3.inChannel.val == 3 - in3.inChannel.val == 4 - in3.inChannel.val == Channel.STOP + in3.inChannel.unwrap() == 3 + in3.inChannel.unwrap() == 4 + in3.inChannel.unwrap() == Channel.STOP } @@ -172,27 +172,27 @@ class ParamsInTest extends Dsl2Spec { in1.name == 'x' in1.filePattern == '*' - in1.inChannel.val == Paths.get('file.x') + in1.inChannel.unwrap() == Paths.get('file.x') in1.index == 0 in2.name == 'f1' in2.filePattern == '*' - in2.inChannel.val == Paths.get('file.x') + in2.inChannel.unwrap() == Paths.get('file.x') in2.index == 1 in3.name == 'f2' in3.filePattern == 'abc' - in3.inChannel.val == Paths.get('file.x') + in3.inChannel.unwrap() == Paths.get('file.x') in3.index == 2 in4.name == 'f3' in4.filePattern == '*.fa' - in4.inChannel.val == Paths.get('file.x') + in4.inChannel.unwrap() == Paths.get('file.x') in4.index == 3 in5.name == 'file.txt' in5.filePattern == 'file.txt' - in5.inChannel.val == Paths.get('file.x') + in5.inChannel.unwrap() == Paths.get('file.x') in5.index == 4 } @@ -228,15 +228,15 @@ class ParamsInTest extends Dsl2Spec { in1.name == '__$fileinparam<0>' in1.getFilePattern(ctx) == 'main.txt' - in1.inChannel.val == Paths.get('file.txt') + in1.inChannel.unwrap() == Paths.get('file.txt') in2.name == '__$fileinparam<1>' in2.getFilePattern(ctx) == 'hello.txt' - in2.inChannel.val == "str" + in2.inChannel.unwrap() == "str" in3.name == 'f2' in3.getFilePattern(ctx) == 'the_file_name.fa' - in3.inChannel.val == Paths.get('file.txt') + in3.inChannel.unwrap() == Paths.get('file.txt') } @@ -273,19 +273,19 @@ class ParamsInTest extends Dsl2Spec { in1.name == '__$fileinparam<0>' in1.getFilePattern(ctx) == 'main.txt' - in1.inChannel.val == Paths.get('file.txt') + in1.inChannel.unwrap() == Paths.get('file.txt') in2.name == '__$fileinparam<1>' in2.getFilePattern(ctx) == 'hello.txt' - in2.inChannel.val == "str" + in2.inChannel.unwrap() == "str" in3.name == 'f2' in3.getFilePattern(ctx) == 'the_file_name.fa' - in3.inChannel.val == Paths.get('file.txt') + in3.inChannel.unwrap() == Paths.get('file.txt') in4.name == 'f3' in4.getFilePattern(ctx) == 'the_file_name.txt' - in4.inChannel.val == Paths.get('file.txt') + in4.inChannel.unwrap() == Paths.get('file.txt') } @@ -318,11 +318,11 @@ class ParamsInTest extends Dsl2Spec { in1.class == StdInParam in1.name == '-' - in1.inChannel.val == 'Hola mundo' + in1.inChannel.unwrap() == 'Hola mundo' in2.class == StdInParam in2.name == '-' - in2.inChannel.val == 'Ciao mondo' + in2.inChannel.unwrap() == 'Ciao mondo' } def testInputEnv() { @@ -354,13 +354,13 @@ class ParamsInTest extends Dsl2Spec { in1.class == EnvInParam in1.name == 'VAR_X' - in1.inChannel.val == 'aaa' + in1.inChannel.unwrap() == 'aaa' in2.class == EnvInParam in2.name == 'VAR_Y' - in2.inChannel.val == 1 - in2.inChannel.val == 2 - in2.inChannel.val == Channel.STOP + in2.inChannel.unwrap() == 1 + in2.inChannel.unwrap() == 2 + in2.inChannel.unwrap() == Channel.STOP } @@ -402,7 +402,7 @@ class ParamsInTest extends Dsl2Spec { in1.inner.get(0).index == 0 in1.inner.get(0).mapIndex == 0 in1.inner.get(0).name == 'p' - in1.inChannel.val == 'Hola mundo' + in1.inChannel.unwrap() == 'Hola mundo' in2.inner.size() == 2 in2.inner.get(0) instanceof ValueInParam @@ -413,7 +413,7 @@ class ParamsInTest extends Dsl2Spec { in2.inner.get(1).name == 'q' in2.inner.get(1).index == 1 in2.inner.get(1).mapIndex == 1 - in2.inChannel.val == 'Hola mundo' + in2.inChannel.unwrap() == 'Hola mundo' in3.inner.size() == 2 in3.inner.get(0) instanceof ValueInParam @@ -425,7 +425,7 @@ class ParamsInTest extends Dsl2Spec { in3.inner.get(1).filePattern == 'file_name.fa' in3.inner.get(1).index == 2 in3.inner.get(1).mapIndex == 1 - in3.inChannel.val == 'str' + in3.inChannel.unwrap() == 'str' in4.inner.size() == 3 in4.inner.get(0) instanceof ValueInParam @@ -441,7 +441,7 @@ class ParamsInTest extends Dsl2Spec { in4.inner.get(2).name == '-' in4.inner.get(2).index == 3 in4.inner.get(2).mapIndex == 2 - in4.inChannel.val == 'ciao' + in4.inChannel.unwrap() == 'ciao' in5.inner.size() == 2 in5.inner.get(0) instanceof ValueInParam @@ -453,7 +453,7 @@ class ParamsInTest extends Dsl2Spec { in5.inner.get(1).filePattern == 'file.fa' in5.inner.get(1).index == 4 in5.inner.get(1).mapIndex == 1 - in5.inChannel.val == 0 + in5.inChannel.unwrap() == 0 } @@ -493,7 +493,7 @@ class ParamsInTest extends Dsl2Spec { def ctx = [x:'the_file', str: 'fastq'] then: - in0.inChannel.val == 'the file content' + in0.inChannel.unwrap() == 'the file content' in0.inner[0] instanceof FileInParam (in0.inner[0] as FileInParam).name == 'name_$x' (in0.inner[0] as FileInParam).getFilePattern(ctx) == 'name_$x' @@ -546,7 +546,7 @@ class ParamsInTest extends Dsl2Spec { process.config.getInputs().size() == 3 in0.name == '__$tupleinparam<0>' - in0.inChannel.val == 1 + in0.inChannel.unwrap() == 1 in0.inner.size() == 3 in0.inner.get(0) instanceof ValueInParam in0.inner.get(0).name == 'a' @@ -629,21 +629,21 @@ class ParamsInTest extends Dsl2Spec { in0.class == EachInParam in0.inChannel instanceof DataflowVariable - in0.inChannel.val == ['aaa'] + in0.inChannel.unwrap() == ['aaa'] in0.inner.name == 'x' in0.inner.owner == in0 in1.class == EachInParam in1.name == '__$eachinparam<1>' in1.inChannel instanceof DataflowVariable - in1.inChannel.val == [1,2] + in1.inChannel.unwrap() == [1,2] in1.inner.name == 'p' in1.inner instanceof ValueInParam in1.inner.owner == in1 in2.class == EachInParam in2.name == '__$eachinparam<2>' - in2.inChannel.val == [1,2,3] + in2.inChannel.unwrap() == [1,2,3] in2.inner instanceof ValueInParam in2.inner.name == 'z' in2.inner.owner == in2 @@ -651,7 +651,7 @@ class ParamsInTest extends Dsl2Spec { in3.class == EachInParam in3.name == '__$eachinparam<3>' in3.inChannel instanceof DataflowVariable - in3.inChannel.val == ['file-a.txt'] + in3.inChannel.unwrap() == ['file-a.txt'] in3.inner instanceof FileInParam in3.inner.name == 'foo' in3.inner.owner == in3 @@ -659,7 +659,7 @@ class ParamsInTest extends Dsl2Spec { in4.class == EachInParam in4.name == '__$eachinparam<4>' in4.inChannel instanceof DataflowVariable - in4.inChannel.val == ['file-x.fa'] + in4.inChannel.unwrap() == ['file-x.fa'] in4.inner instanceof FileInParam in4.inner.name == 'bar' in4.inner.filePattern == 'bar' @@ -735,28 +735,28 @@ class ParamsInTest extends Dsl2Spec { in0.name == 'x' in0.filePattern == '*' - in0.inChannel.val == FILE + in0.inChannel.unwrap() == FILE in0.index == 0 in0.isPathQualifier() in0.arity == new ArityParam.Range(1, 1) in1.name == 'f1' in1.filePattern == '*' - in1.inChannel.val == FILE + in1.inChannel.unwrap() == FILE in1.index == 1 in1.isPathQualifier() in1.arity == new ArityParam.Range(1, 2) in2.name == '*.fa' in2.filePattern == '*.fa' - in2.inChannel.val == FILE + in2.inChannel.unwrap() == FILE in2.index == 2 in2.isPathQualifier() in2.arity == new ArityParam.Range(1, Integer.MAX_VALUE) in3.name == 'file.txt' in3.filePattern == 'file.txt' - in3.inChannel.val == FILE + in3.inChannel.unwrap() == FILE in3.index == 3 in3.isPathQualifier() @@ -800,12 +800,12 @@ class ParamsInTest extends Dsl2Spec { in1.name == '__$pathinparam<0>' in1.getFilePattern(ctx) == 'main.txt' - in1.inChannel.val == 'file.txt' + in1.inChannel.unwrap() == 'file.txt' in1.isPathQualifier() in2.name == '__$pathinparam<1>' in2.getFilePattern(ctx) == 'hello.txt' - in2.inChannel.val == "str" + in2.inChannel.unwrap() == "str" in2.isPathQualifier() } @@ -835,7 +835,7 @@ class ParamsInTest extends Dsl2Spec { def ctx = [x:'the_file', str: 'fastq'] then: - in1.inChannel.val == '/the/file/path' + in1.inChannel.unwrap() == '/the/file/path' in1.inner[0] instanceof FileInParam (in1.inner[0] as FileInParam).getName() == '__$pathinparam<0:0>' (in1.inner[0] as FileInParam).getFilePattern(ctx) == 'hola_the_file' @@ -875,7 +875,7 @@ class ParamsInTest extends Dsl2Spec { process.config.getInputs().size() == 3 in0.name == '__$tupleinparam<0>' - in0.inChannel.val == 1 + in0.inChannel.unwrap() == 1 in0.inner.size() == 2 in0.inner.get(0) instanceof ValueInParam in0.inner.get(0).name == 'a' @@ -938,7 +938,7 @@ class ParamsInTest extends Dsl2Spec { in0.class == EachInParam in0.name == '__$eachinparam<0>' in0.inChannel instanceof DataflowVariable - in0.inChannel.val == ['file-a.txt'] + in0.inChannel.unwrap() == ['file-a.txt'] in0.inner instanceof FileInParam (in0.inner as FileInParam).name == 'foo' (in0.inner as FileInParam).owner == in0 @@ -947,7 +947,7 @@ class ParamsInTest extends Dsl2Spec { in1.class == EachInParam in1.name == '__$eachinparam<1>' in1.inChannel instanceof DataflowVariable - in1.inChannel.val == ['file-x.fa'] + in1.inChannel.unwrap() == ['file-x.fa'] in1.inner instanceof FileInParam (in1.inner as FileInParam).name == 'bar' (in1.inner as FileInParam).filePattern == 'bar' From 1826cfdb1197ab8eeb7c8d2cefb815fc1af111f6 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 7 Jan 2025 23:50:31 +0800 Subject: [PATCH 09/66] Fix tests Signed-off-by: Paolo Di Tommaso --- .../executor/BashWrapperBuilderTest.groovy | 20 +++++++++++++++++-- .../executor/test-bash-wrapper-with-trace.txt | 1 + .../nextflow/executor/test-bash-wrapper.txt | 1 + 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/modules/nextflow/src/test/groovy/nextflow/executor/BashWrapperBuilderTest.groovy b/modules/nextflow/src/test/groovy/nextflow/executor/BashWrapperBuilderTest.groovy index e54bd97d72..61f72eb3f5 100644 --- a/modules/nextflow/src/test/groovy/nextflow/executor/BashWrapperBuilderTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/executor/BashWrapperBuilderTest.groovy @@ -27,6 +27,7 @@ import nextflow.container.ContainerConfig import nextflow.container.DockerBuilder import nextflow.container.SingularityBuilder import nextflow.processor.TaskBean +import nextflow.processor.TaskId import nextflow.util.MustacheTemplateEngine import org.yaml.snakeyaml.Yaml import spock.lang.Specification @@ -262,6 +263,7 @@ class BashWrapperBuilderTest extends Specification { */ when: def wrapper = newBashWrapperBuilder( + taskId: TaskId.of(100), name: 'Hello 1', workDir: folder, headerScript: '#BSUB -x 1\n#BSUB -y 2', @@ -286,6 +288,7 @@ class BashWrapperBuilderTest extends Specification { */ when: def wrapper = newBashWrapperBuilder( + taskId: TaskId.of(200), name: 'Hello 2', workDir: folder, statsEnabled: true ) .buildNew0() @@ -342,13 +345,15 @@ class BashWrapperBuilderTest extends Specification { def 'should create task metadata string' () { given: def builder = newBashWrapperBuilder( + taskId: TaskId.of(100), name: 'foo', arrayIndexName: 'SLURM_ARRAY_TASK_ID', arrayIndexStart: 0, arrayWorkDirs: [ Path.of('/work/01'), Path.of('/work/02'), Path.of('/work/03') ], containerConfig: [enabled: true], containerImage: 'quay.io/nextflow:bash', - outputFiles: ['foo.txt', '*.bar', '**/baz'] + outputFiles: ['foo.txt', '*.bar', '**/baz'], + upstreamTasks: new LinkedHashSet<>([TaskId.of(10), TaskId.of(20)]) ) when: @@ -356,6 +361,7 @@ class BashWrapperBuilderTest extends Specification { then: meta == '''\ ### --- + ### id: '100' ### name: 'foo' ### array: ### index-name: SLURM_ARRAY_TASK_ID @@ -369,6 +375,9 @@ class BashWrapperBuilderTest extends Specification { ### - 'foo.txt' ### - '*.bar' ### - '**/baz' + ### upstream-tasks: + ### - '10' + ### - '20' ### ... '''.stripIndent() @@ -376,6 +385,7 @@ class BashWrapperBuilderTest extends Specification { def yaml = meta.readLines().collect(it-> it.substring(4)).join('\n') def obj = new Yaml().load(yaml) as Map then: + obj.id == '100' obj.name == 'foo' obj.array == [ 'index-name':'SLURM_ARRAY_TASK_ID', @@ -388,17 +398,22 @@ class BashWrapperBuilderTest extends Specification { def 'should add task metadata' () { when: - def bash = newBashWrapperBuilder([name:'task1']) + def bash = newBashWrapperBuilder( + taskId: TaskId.of(123), + name:'task1' + ) then: bash.makeBinding().containsKey('task_metadata') bash.makeBinding().task_metadata == '''\ ### --- + ### id: '123' ### name: 'task1' ### ... '''.stripIndent() when: bash = newBashWrapperBuilder( + taskId: TaskId.of(321), name: 'task2', arrayIndexName: 'SLURM_ARRAY_TASK_ID', arrayIndexStart: 0, @@ -410,6 +425,7 @@ class BashWrapperBuilderTest extends Specification { then: bash.makeBinding().task_metadata == '''\ ### --- + ### id: '321' ### name: 'task2' ### array: ### index-name: SLURM_ARRAY_TASK_ID diff --git a/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper-with-trace.txt b/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper-with-trace.txt index 5aef2f4795..c4bbe2e8ba 100644 --- a/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper-with-trace.txt +++ b/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper-with-trace.txt @@ -1,5 +1,6 @@ #!/bin/bash ### --- +### id: '200' ### name: 'Hello 2' ### ... set -e diff --git a/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper.txt b/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper.txt index 3bb4f34fe5..2025790233 100644 --- a/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper.txt +++ b/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper.txt @@ -2,6 +2,7 @@ #BSUB -x 1 #BSUB -y 2 ### --- +### id: '100' ### name: 'Hello 1' ### ... set -e From e7c8722de1367090b94e75f7fff26311f0dcd515 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Wed, 8 Jan 2025 04:57:45 +0800 Subject: [PATCH 10/66] Fix more tests Signed-off-by: Paolo Di Tommaso --- .../nextflow/script/ScriptPipesTest.groovy | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/modules/nextflow/src/test/groovy/nextflow/script/ScriptPipesTest.groovy b/modules/nextflow/src/test/groovy/nextflow/script/ScriptPipesTest.groovy index 20c44c2525..db38b87a36 100644 --- a/modules/nextflow/src/test/groovy/nextflow/script/ScriptPipesTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/script/ScriptPipesTest.groovy @@ -43,8 +43,8 @@ class ScriptPipesTest extends Dsl2Spec { def result = runner.setScript(SCRIPT).execute() then: - result[0].val == 'olleH mundo' - result[1].val == 'OLLEH' + result[0].unwrap() == 'olleH mundo' + result[1].unwrap() == 'OLLEH' } @@ -75,7 +75,7 @@ class ScriptPipesTest extends Dsl2Spec { def result = runner.setScript(SCRIPT).execute() then: - result.val == 'DLROW ALOH' + result.unwrap() == 'DLROW ALOH' } def 'should pipe multi-outputs with multi-inputs' () { @@ -114,7 +114,7 @@ class ScriptPipesTest extends Dsl2Spec { def result = runner.setScript(SCRIPT).execute() then: - result.val == 'olleh + HELLO' + result.unwrap() == 'olleh + HELLO' } def 'should pipe process with operator' () { @@ -145,9 +145,9 @@ class ScriptPipesTest extends Dsl2Spec { def result = runner.setScript(SCRIPT).execute() then: - result.val == 'aloh' - result.val == 'HOLA' - result.val == 'hola' + result.unwrap() == 'aloh' + result.unwrap() == 'HOLA' + result.unwrap() == 'hola' } @@ -171,7 +171,7 @@ class ScriptPipesTest extends Dsl2Spec { def result = runner.setScript(SCRIPT).execute() then: - result.val == 'aloh' + result.unwrap() == 'aloh' } @@ -201,7 +201,7 @@ class ScriptPipesTest extends Dsl2Spec { def result = runner.setScript(SCRIPT).execute() then: - result.val == 'HOLA' + result.unwrap() == 'HOLA' } @@ -225,7 +225,7 @@ class ScriptPipesTest extends Dsl2Spec { def result = runner.setScript(SCRIPT).execute() then: - result.val.sort() == [1, 4, 9] + result.unwrap().sort() == [1, 4, 9] } @@ -238,9 +238,9 @@ class ScriptPipesTest extends Dsl2Spec { when: def result = new MockScriptRunner().setScript(SCRIPT).execute() then: - result.val == 10 - result.val == 20 - result.val == 30 + result.unwrap() == 10 + result.unwrap() == 20 + result.unwrap() == 30 } @@ -262,7 +262,7 @@ class ScriptPipesTest extends Dsl2Spec { when: def result = new MockScriptRunner().setScript(SCRIPT).execute() then: - result.val == 40 + result.unwrap() == 40 } def 'should compose custom funs' () { @@ -285,7 +285,7 @@ class ScriptPipesTest extends Dsl2Spec { when: def result = new MockScriptRunner().setScript(SCRIPT).execute() then: - result.val == 22 + result.unwrap() == 22 } @@ -312,7 +312,7 @@ class ScriptPipesTest extends Dsl2Spec { when: def result = new MockScriptRunner().setScript(SCRIPT).execute() then: - result.val == [1,2] + result.unwrap() == [1,2] } @@ -340,7 +340,7 @@ class ScriptPipesTest extends Dsl2Spec { when: def result = new MockScriptRunner().setScript(SCRIPT).execute() then: - result.val == 'hi'.reverse() + result.unwrap() == 'hi'.reverse() } @@ -368,7 +368,7 @@ class ScriptPipesTest extends Dsl2Spec { when: def result = new MockScriptRunner().setScript(SCRIPT).execute() then: - result.val == 'hello'.reverse() + result.unwrap() == 'hello'.reverse() } @@ -400,7 +400,7 @@ class ScriptPipesTest extends Dsl2Spec { when: def result = new MockScriptRunner().setScript(SCRIPT).execute() then: - result.val == 'HI'.reverse() + result.unwrap() == 'HI'.reverse() } @@ -430,7 +430,7 @@ class ScriptPipesTest extends Dsl2Spec { when: def result = new MockScriptRunner().setScript(SCRIPT).execute() then: - result.val == 33 + result.unwrap() == 33 cleanup: From 88ae57e0310aa3b9cd7a7611099e5a26a597fd27 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Wed, 8 Jan 2025 13:00:29 +0400 Subject: [PATCH 11/66] Add chain operator Signed-off-by: Paolo Di Tommaso --- .../groovy/nextflow/extension/ChainOp.groovy | 95 +++++++++++++++++++ .../nextflow/extension/DataflowHelper.groovy | 15 --- .../nextflow/extension/OperatorImpl.groovy | 26 +++-- .../groovy/nextflow/extension/ReduceOp.groovy | 12 +-- 4 files changed, 116 insertions(+), 32 deletions(-) create mode 100644 modules/nextflow/src/main/groovy/nextflow/extension/ChainOp.groovy diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/ChainOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/ChainOp.groovy new file mode 100644 index 0000000000..6252b7e2ae --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/extension/ChainOp.groovy @@ -0,0 +1,95 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.extension + +import groovy.transform.CompileStatic +import groovyx.gpars.dataflow.DataflowReadChannel +import groovyx.gpars.dataflow.DataflowWriteChannel +import groovyx.gpars.dataflow.operator.ChainWithClosure +import groovyx.gpars.dataflow.operator.DataflowEventListener + +import static nextflow.extension.DataflowHelper.newOperator +import static nextflow.extension.DataflowHelper.OpParams + +/** + * Implements the chain operator + * + * @author Paolo Di Tommaso + */ +@CompileStatic +class ChainOp { + + private DataflowReadChannel source + private DataflowWriteChannel target + private List listeners = List.of() + private boolean accumulator + private Closure action + + static ChainOp create() { + new ChainOp() + } + + ChainOp withSource(DataflowReadChannel source) { + assert source + this.source = source + return this + } + + ChainOp withTarget(DataflowWriteChannel target) { + assert target + this.target = target + return this + } + + ChainOp withListener(DataflowEventListener listener) { + assert listener != null + this.listeners = List.of(listener) + return this + } + + ChainOp withListeners(List listeners) { + assert listeners != null + this.listeners = listeners + return this + } + + ChainOp withAccumulator(boolean value) { + this.accumulator = value + return this + } + + ChainOp withAction(Closure action) { + this.action = action + return this + } + + DataflowWriteChannel apply() { + assert source + assert target + assert action + + final OpParams parameters = new OpParams() + .withInput(source) + .withOutput(target) + .withAccumulator(accumulator) + .withListeners(listeners) + + newOperator(parameters, new ChainWithClosure(action)) + return target + } +} diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy index 5b68f56c22..9be3348851 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy @@ -28,7 +28,6 @@ import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowVariable import groovyx.gpars.dataflow.DataflowWriteChannel import groovyx.gpars.dataflow.expression.DataflowExpression -import groovyx.gpars.dataflow.operator.ChainWithClosure import groovyx.gpars.dataflow.operator.DataflowEventAdapter import groovyx.gpars.dataflow.operator.DataflowEventListener import groovyx.gpars.dataflow.operator.DataflowProcessor @@ -364,20 +363,6 @@ class DataflowHelper { } } - @Deprecated - static DataflowProcessor chainImpl(final DataflowReadChannel source, final DataflowWriteChannel target, final Map params, final Closure closure) { - - final OpParams parameters = new OpParams() - .withInput(source) - .withOutput(target) - - newOperator(parameters, new ChainWithClosure(closure)) - } - - static DataflowProcessor chainImpl(OpParams params, final Closure closure) { - newOperator(params, new ChainWithClosure(closure)) - } - @PackageScope @CompileStatic static KeyPair makeKey(List pivot, entry) { diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy index e0163cd14d..4ed29e7306 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy @@ -106,9 +106,11 @@ class OperatorImpl { * @return */ DataflowWriteChannel chain(final DataflowReadChannel source, final Map params, final Closure closure) { - final target = CH.createBy(source) - chainImpl(source, target, params, closure) - return target; + return ChainOp.create() + .withSource(source) + .withTarget(CH.createBy(source)) + .withAction(closure) + .apply() } /** @@ -344,10 +346,12 @@ class OperatorImpl { } } as Closure - // filter removing all duplicates - chainImpl(source, target, [listeners: [events]], filter ) - - return target + return ChainOp.create() + .withSource(source) + .withTarget(target) + .withListener(events) + .withAction(filter) + .apply() } /** @@ -383,9 +387,11 @@ class OperatorImpl { return it } - chainImpl(source, target, [:], filter) - - return target + return ChainOp.create() + .withSource(source) + .withTarget(target) + .withAction(filter) + .apply() } /** diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/ReduceOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/ReduceOp.groovy index 00156d7d7c..7a740e068b 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/ReduceOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/ReduceOp.groovy @@ -26,9 +26,6 @@ import groovyx.gpars.dataflow.operator.DataflowProcessor import nextflow.Channel import nextflow.Global import nextflow.Session - -import static nextflow.extension.DataflowHelper.chainImpl - /** * Implements reduce operator logic * @@ -130,12 +127,13 @@ class ReduceOp { } } - final params = new DataflowHelper.OpParams() - .withInput(source) - .withOutput(CH.create()) + ChainOp.create() + .withSource(source) + .withTarget(CH.create()) .withListener(listener) .withAccumulator(true) - chainImpl(params, {true}) + .withAction({true}) + .apply() return target } From c96d5734604ac2e4ff5cd53e426390c8a4d8fd80 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Wed, 8 Jan 2025 13:09:30 +0400 Subject: [PATCH 12/66] Fix tests Signed-off-by: Paolo Di Tommaso --- .../nextflow/processor/TaskProcessor.groovy | 4 ++-- .../processor/TaskProcessorTest.groovy | 18 +++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy index 19ce31fd8d..867ebdb3d8 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy @@ -136,8 +136,8 @@ class TaskProcessor { @Canonical static class FairEntry { - TaskRun task Map emissions + TaskRun task } static final public String TASK_CONTEXT_PROPERTY_NAME = 'task' @@ -1484,7 +1484,7 @@ class TaskProcessor { synchronized (isFair0) { // decrement -1 because tasks are 1-based final index = task.index-1 - FairEntry entry = new FairEntry(task,emissions) + FairEntry entry = new FairEntry(emissions, task) // store the task emission values in a buffer fairBuffers[index-currentEmission] = entry // check if the current task index matches the expected next emission index diff --git a/modules/nextflow/src/test/groovy/nextflow/processor/TaskProcessorTest.groovy b/modules/nextflow/src/test/groovy/nextflow/processor/TaskProcessorTest.groovy index 751feeb03f..814ecf8a68 100644 --- a/modules/nextflow/src/test/groovy/nextflow/processor/TaskProcessorTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/processor/TaskProcessorTest.groovy @@ -901,40 +901,40 @@ class TaskProcessorTest extends Specification { when: processor.fairBindOutputs0(emission3, task3) then: - processor.@fairBuffers[2] == emission3 + processor.@fairBuffers[2] == new TaskProcessor.FairEntry(emission3,task3) 0 * processor.bindOutputs0(_) when: processor.fairBindOutputs0(emission2, task2) then: - processor.@fairBuffers[1] == emission2 + processor.@fairBuffers[1] == new TaskProcessor.FairEntry(emission2,task2) 0 * processor.bindOutputs0(_) when: processor.fairBindOutputs0(emission5, task5) then: - processor.@fairBuffers[4] == emission5 + processor.@fairBuffers[4] == new TaskProcessor.FairEntry(emission5, task5) 0 * processor.bindOutputs0(_) when: processor.fairBindOutputs0(emission1, task1) then: - 1 * processor.bindOutputs0(emission1) + 1 * processor.bindOutputs0(emission1, task1) then: - 1 * processor.bindOutputs0(emission2) + 1 * processor.bindOutputs0(emission2, task2) then: - 1 * processor.bindOutputs0(emission3) + 1 * processor.bindOutputs0(emission3, task3) and: processor.@fairBuffers.size() == 2 processor.@fairBuffers[0] == null - processor.@fairBuffers[1] == emission5 + processor.@fairBuffers[1] == new TaskProcessor.FairEntry(emission5, task5) when: processor.fairBindOutputs0(emission4, task4) then: - 1 * processor.bindOutputs0(emission4) + 1 * processor.bindOutputs0(emission4, task4) then: - 1 * processor.bindOutputs0(emission5) + 1 * processor.bindOutputs0(emission5, task5) then: processor.@fairBuffers.size()==0 } From f7270057bd2e87b3df27cce6e4d4068e716bd159 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Thu, 9 Jan 2025 14:34:59 +0100 Subject: [PATCH 13/66] Fix and improvements Signed-off-by: Paolo Di Tommaso --- .../groovy/nextflow/extension/BranchOp.groovy | 13 +- .../nextflow/extension/CollectFileOp.groovy | 10 +- .../nextflow/extension/CollectOp.groovy | 6 +- .../groovy/nextflow/extension/ConcatOp.groovy | 9 +- .../nextflow/extension/DataflowHelper.groovy | 26 ++- .../groovy/nextflow/extension/FilterOp.groovy | 79 +++++++++ .../groovy/nextflow/extension/MapOp.groovy | 4 +- .../main/groovy/nextflow/extension/Op.groovy | 162 ------------------ .../nextflow/extension/OperatorImpl.groovy | 58 ++----- .../groovy/nextflow/extension/ReduceOp.groovy | 4 +- .../groovy/nextflow/extension/ToListOp.groovy | 4 +- .../groovy/nextflow/extension/op/Op.groovy | 83 +++++++++ .../extension/op/OpAccumulatorClosure.groovy | 42 +++++ .../nextflow/extension/op/OpClosure.groovy | 126 ++++++++++++++ .../nextflow/extension/op/OpContext.groovy | 29 ++++ .../main/groovy/nextflow/prov/Tracker.groovy | 33 ++-- .../main/groovy/nextflow/prov/TrailRun.groovy | 1 + .../groovy/nextflow/extension/OpTest.groovy | 8 +- .../extension/op/OpClosureTest.groovy | 41 +++++ .../test/groovy/nextflow/prov/ProvTest.groovy | 27 +++ 20 files changed, 516 insertions(+), 249 deletions(-) create mode 100644 modules/nextflow/src/main/groovy/nextflow/extension/FilterOp.groovy delete mode 100644 modules/nextflow/src/main/groovy/nextflow/extension/Op.groovy create mode 100644 modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy create mode 100644 modules/nextflow/src/main/groovy/nextflow/extension/op/OpAccumulatorClosure.groovy create mode 100644 modules/nextflow/src/main/groovy/nextflow/extension/op/OpClosure.groovy create mode 100644 modules/nextflow/src/main/groovy/nextflow/extension/op/OpContext.groovy create mode 100644 modules/nextflow/src/test/groovy/nextflow/extension/op/OpClosureTest.groovy diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/BranchOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/BranchOp.groovy index 868db7131f..082da3c2b7 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/BranchOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/BranchOp.groovy @@ -20,7 +20,9 @@ import groovy.transform.CompileStatic import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowWriteChannel import groovyx.gpars.dataflow.expression.DataflowExpression +import groovyx.gpars.dataflow.operator.DataflowProcessor import nextflow.Channel +import nextflow.extension.op.Op import nextflow.script.ChannelOut import nextflow.script.TokenBranchChoice import nextflow.script.TokenBranchDef @@ -50,20 +52,21 @@ class BranchOp { ChannelOut getOutput() { this.output } - protected void doNext(it) { + protected void doNext(DataflowProcessor proc, Object it) { TokenBranchChoice ret = switchDef.closure.call(it) if( ret ) { - Op.bind(targets[ret.choice], ret.value) + Op.bind(proc, targets[ret.choice], ret.value) } } - protected void doComplete(nope) { + protected void doComplete(DataflowProcessor proc) { for( DataflowWriteChannel ch : targets.values() ) { if( ch instanceof DataflowExpression ) { - if( !ch.isBound()) ch.bind(Channel.STOP) + if( !ch.isBound() ) + Op.bind(proc, ch, Channel.STOP) } else { - ch.bind(Channel.STOP) + Op.bind(proc, ch, Channel.STOP) } } } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/CollectFileOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/CollectFileOp.groovy index 4b8c181d50..2627d9147d 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/CollectFileOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/CollectFileOp.groovy @@ -24,8 +24,10 @@ import java.nio.file.Path import groovy.util.logging.Slf4j import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowWriteChannel +import groovyx.gpars.dataflow.operator.DataflowProcessor import nextflow.Channel import nextflow.Global +import nextflow.extension.op.Op import nextflow.file.FileCollector import nextflow.file.FileHelper import nextflow.file.SimpleFileCollector @@ -137,7 +139,7 @@ class CollectFileOp { * each time a value is received, invoke the closure and * append its result value to a file */ - protected processItem( item ) { + protected processItem( DataflowProcessor proc, Object item ) { def value = closure ? closure.call(item) : item // when the value is a list, the first item hold the grouping key @@ -183,13 +185,13 @@ class CollectFileOp { * * @params obj: NOT USED. It needs to be declared because this method is invoked as a closure */ - protected emitItems( obj ) { + protected emitItems(DataflowProcessor processor) { // emit collected files to 'result' channel collector.saveTo(storeDir).each { - Op.bind(result,it) + Op.bind(processor, result, it) } // close the channel - Op.bind(result,Channel.STOP) + Op.bind(processor, result, Channel.STOP) // close the collector collector.safeClose() } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/CollectOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/CollectOp.groovy index 71cdf5aedb..a75fc16fa4 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/CollectOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/CollectOp.groovy @@ -21,7 +21,9 @@ import static nextflow.util.CheckHelper.* import groovy.transform.CompileStatic import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowVariable +import groovyx.gpars.dataflow.operator.DataflowProcessor import nextflow.Channel +import nextflow.extension.op.Op import nextflow.util.ArrayBag /** * Implements {@link OperatorImpl#collect(groovyx.gpars.dataflow.DataflowReadChannel)} operator @@ -55,9 +57,9 @@ class CollectOp { Map events = [:] events.onNext = { append(result, it) } - events.onComplete = { + events.onComplete = { DataflowProcessor processor -> final msg = result ? new ArrayBag(normalise(result)) : Channel.STOP - Op.bind(target, msg) + Op.bind(processor, target, msg) } DataflowHelper.subscribeImpl(source, events) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/ConcatOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/ConcatOp.groovy index 34e9ceca38..e2dd99bea5 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/ConcatOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/ConcatOp.groovy @@ -19,7 +19,10 @@ package nextflow.extension import groovy.transform.CompileStatic import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowWriteChannel +import groovyx.gpars.dataflow.operator.DataflowProcessor import nextflow.Channel +import nextflow.extension.op.Op + /** * Implements the {@link OperatorImpl#concat} operator * @@ -56,10 +59,10 @@ class ConcatOp { def next = index < channels.size() ? channels[index] : null def events = new HashMap(2) - events.onNext = { Op.bind(result, it) } - events.onComplete = { + events.onNext = { DataflowProcessor proc, it -> Op.bind(proc, result, it) } + events.onComplete = { DataflowProcessor proc -> if(next) append(result, channels, index) - else Op.bind(result, Channel.STOP) + else Op.bind(proc, result, Channel.STOP) } DataflowHelper.subscribeImpl(current, events) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy index 9be3348851..4026b4d1d8 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy @@ -30,11 +30,14 @@ import groovyx.gpars.dataflow.DataflowWriteChannel import groovyx.gpars.dataflow.expression.DataflowExpression import groovyx.gpars.dataflow.operator.DataflowEventAdapter import groovyx.gpars.dataflow.operator.DataflowEventListener +import groovyx.gpars.dataflow.operator.DataflowOperator import groovyx.gpars.dataflow.operator.DataflowProcessor import nextflow.Channel import nextflow.Global import nextflow.Session import nextflow.dag.NodeMarker +import nextflow.extension.op.Op + /** * This class provides helper methods to implement nextflow operators * @@ -278,13 +281,17 @@ class DataflowHelper { assert params.listeners // create the underlying dataflow operator - final op = Dataflow.operator(params.toMap(), Op.instrument(code, params.accumulator)) + final closure = Op.instrument(code, params.accumulator) + final group = Dataflow.retrieveCurrentDFPGroup() + final operator = new DataflowOperator(group, params.toMap(), closure) + Op.context.put(operator, closure) + operator.start() // track the operator as dag node - NodeMarker.appendOperator(op) + NodeMarker.appendOperator(operator) if( session && session.allOperators != null ) { - session.allOperators << op + session.allOperators << operator } - return op + return operator } /* @@ -354,11 +361,16 @@ class DataflowHelper { .withAccumulator(accumulator) newOperator (params) { - if( events.onNext ) { - events.onNext.call(it) + final proc = ((DataflowProcessor) getDelegate()) + if( events.onNext instanceof Closure ) { + final action = (Closure) events.onNext + final types = action.getParameterTypes() + types.size()==2 && types[0]==DataflowProcessor.class + ? action.call(proc, it) + : action.call(it) } if( stopOnFirst ) { - ((DataflowProcessor) getDelegate()).terminate() + proc.terminate() } } } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/FilterOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/FilterOp.groovy new file mode 100644 index 0000000000..3e38e82187 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/extension/FilterOp.groovy @@ -0,0 +1,79 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.extension + +import static nextflow.extension.DataflowHelper.* + +import groovyx.gpars.dataflow.DataflowReadChannel +import groovyx.gpars.dataflow.DataflowWriteChannel +import groovyx.gpars.dataflow.expression.DataflowExpression +import groovyx.gpars.dataflow.operator.DataflowProcessor +import nextflow.Channel +import nextflow.extension.op.Op +import org.codehaus.groovy.runtime.callsite.BooleanReturningMethodInvoker +import org.codehaus.groovy.runtime.typehandling.DefaultTypeTransformation +/** + * + * @author Paolo Di Tommaso + */ +class FilterOp { + + private DataflowReadChannel source + + private Object criteria + + FilterOp withSource(DataflowReadChannel source) { + this.source = source + return this + } + + FilterOp withCriteria(Closure criteria) { + this.criteria = criteria + return this + } + + FilterOp withCriteria(Object criteria) { + this.criteria = criteria + return this + } + + DataflowWriteChannel apply() { + assert source!=null + assert criteria!=null + + final discriminator = criteria !instanceof Closure + ? new BooleanReturningMethodInvoker("isCase") + : null + final target = CH.createBy(source) + final stopOnFirst = source instanceof DataflowExpression + newOperator(source, target, { + final result = criteria instanceof Closure + ? DefaultTypeTransformation.castToBoolean(criteria.call(it)) + : discriminator.invoke(criteria, (Object)it) + final proc = ((DataflowProcessor) getDelegate()) + if( result ) { + Op.bind(proc, target, it) + } + if( stopOnFirst ) { + Op.bind(proc, target, Channel.STOP) + } + }) + + return target + } +} diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/MapOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/MapOp.groovy index ecea4fbc2f..78dafff250 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/MapOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/MapOp.groovy @@ -21,6 +21,8 @@ import groovyx.gpars.dataflow.DataflowWriteChannel import groovyx.gpars.dataflow.expression.DataflowExpression import groovyx.gpars.dataflow.operator.DataflowProcessor import nextflow.Channel +import nextflow.extension.op.Op + /** * Implements {@link OperatorImpl#map(groovyx.gpars.dataflow.DataflowReadChannel, groovy.lang.Closure)} operator * @@ -57,7 +59,7 @@ class MapOp { // bind the result value if (result != Channel.VOID) - Op.bind(target, result) + Op.bind(proc, target, result) // when the `map` operator is applied to a dataflow flow variable // terminate the processor after the first emission -- Issue #44 diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/Op.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/Op.groovy deleted file mode 100644 index 5f4a997725..0000000000 --- a/modules/nextflow/src/main/groovy/nextflow/extension/Op.groovy +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Copyright 2013-2024, Seqera Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package nextflow.extension - - -import groovy.transform.CompileStatic -import groovy.transform.PackageScope -import groovy.util.logging.Slf4j -import groovyx.gpars.dataflow.DataflowWriteChannel -import groovyx.gpars.dataflow.operator.PoisonPill -import nextflow.Global -import nextflow.Session -import nextflow.prov.OperatorRun -import nextflow.prov.Prov -import nextflow.prov.Tracker -import org.codehaus.groovy.runtime.InvokerHelper - -/** - * Operator helpers methods - * - * @author Paolo Di Tommaso - */ -@Slf4j -@CompileStatic -class Op { - - static final @PackageScope ThreadLocal currentOperator = new ThreadLocal<>() - - static List unwrap(List messages) { - return messages.collect(it -> it instanceof Tracker.Msg ? it.value : it) - } - - static Object unwrap(Object it) { - return it instanceof Tracker.Msg ? it.value : it - } - - static Tracker.Msg wrap(Object obj) { - obj instanceof Tracker.Msg ? obj : Tracker.Msg.of(obj) - } - - static void bind(DataflowWriteChannel channel, Object msg) { - try { - if( msg instanceof PoisonPill ) - channel.bind(msg) - else - Prov.getTracker().bindOutput(currentOperator.get(), channel, msg) - } - catch (Throwable t) { - log.error("Unexpected resolving execution provenance: ${t.message}", t) - (Global.session as Session).abort(t) - } - } - - static Closure instrument(Closure op, boolean accumulator=false) { - return new InvokeOperatorAdapter(op, accumulator) - } - - static class InvokeOperatorAdapter extends Closure { - - private final Closure target - - private final boolean accumulator - - private OperatorRun previousRun - - private InvokeOperatorAdapter(Closure code, boolean accumulator) { - super(code.owner, code.thisObject) - this.target = code - this.target.delegate = code.delegate - this.target.setResolveStrategy(code.resolveStrategy) - this.accumulator = accumulator - } - - @Override - Class[] getParameterTypes() { - return target.getParameterTypes() - } - - @Override - int getMaximumNumberOfParameters() { - return target.getMaximumNumberOfParameters() - } - - @Override - Object getDelegate() { - return target.getDelegate() - } - - @Override - Object getProperty(String propertyName) { - return target.getProperty(propertyName) - } - - @Override - int getDirective() { - return target.getDirective() - } - - @Override - void setDelegate(Object delegate) { - target.setDelegate(delegate) - } - - @Override - void setDirective(int directive) { - target.setDirective(directive) - } - - @Override - void setResolveStrategy(int resolveStrategy) { - target.setResolveStrategy(resolveStrategy) - } - - @Override - void setProperty(String propertyName, Object newValue) { - target.setProperty(propertyName, newValue) - } - - @Override - Object call(final Object... args) { - // when the accumulator flag true, re-use the previous run object - final run = !accumulator || previousRun==null - ? new OperatorRun() - : previousRun - // set as the current run in the thread local - currentOperator.set(run) - // map the inputs - final inputs = Prov.getTracker().receiveInputs(run, args.toList()) - final ret = InvokerHelper.invokeMethod(target, 'call', inputs.toArray()) - // track the previous run - if( accumulator ) - previousRun = run - // return the operation result - return ret - } - - Object call(Object args) { - call(InvokerHelper.asArray(args)) - } - - @Override - Object call() { - call(InvokerHelper.EMPTY_ARGS) - } - } - -} diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy index 4ed29e7306..686d0462d1 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy @@ -36,6 +36,7 @@ import nextflow.Channel import nextflow.Global import nextflow.NF import nextflow.Session +import nextflow.extension.op.Op import nextflow.script.ChannelOut import nextflow.script.TokenBranchDef import nextflow.script.TokenMultiMapDef @@ -44,7 +45,6 @@ import nextflow.splitter.FastqSplitter import nextflow.splitter.JsonSplitter import nextflow.splitter.TextSplitter import org.codehaus.groovy.runtime.callsite.BooleanReturningMethodInvoker -import org.codehaus.groovy.runtime.typehandling.DefaultTypeTransformation /** * A set of operators inspired to RxJava extending the methods available on DataflowChannel * data structure @@ -139,33 +139,34 @@ class OperatorImpl { final target = CH.create() final listener = stopErrorListener(source,target) - newOperator(source, target, listener) { item -> + newOperator(source, target, listener) { Object item -> final result = closure != null ? closure.call(item) : item + final proc = ((DataflowProcessor) getDelegate()) switch( result ) { case Collection: - result.each { it -> Op.bind(target,it) } + result.each { it -> Op.bind(proc, target,it) } break case (Object[]): - result.each { it -> Op.bind(target,it) } + result.each { it -> Op.bind(proc, target,it) } break case Map: - result.each { it -> Op.bind(target,it) } + result.each { it -> Op.bind(proc, target,it) } break case Map.Entry: - Op.bind(target, (result as Map.Entry).key ) - Op.bind(target, (result as Map.Entry).value ) + Op.bind(proc, target, (result as Map.Entry).key ) + Op.bind(proc, target, (result as Map.Entry).value ) break case Channel.VOID: break default: - Op.bind(target,result) + Op.bind(proc, target, result) } } @@ -252,48 +253,23 @@ class OperatorImpl { * @return */ DataflowWriteChannel filter(final DataflowReadChannel source, final Object criteria) { - def discriminator = new BooleanReturningMethodInvoker("isCase"); - - def target = CH.createBy(source) - if( source instanceof DataflowExpression ) { - source.whenBound { - def result = it instanceof ControlMessage ? false : discriminator.invoke(criteria, (Object)it) - target.bind( result ? it : Channel.STOP ) - } - } - else { - newOperator(source, target, { - def result = discriminator.invoke(criteria, (Object)it) - if( result ) target.bind(it) - }) - } - - return target + return new FilterOp() + .withSource(source) + .withCriteria(criteria) + .apply() } DataflowWriteChannel filter(DataflowReadChannel source, final Closure closure) { - def target = CH.createBy(source) - if( source instanceof DataflowExpression ) { - source.whenBound { - def result = it instanceof ControlMessage ? false : DefaultTypeTransformation.castToBoolean(closure.call(it)) - target.bind( result ? it : Channel.STOP ) - } - } - else { - newOperator(source, target, { - def result = DefaultTypeTransformation.castToBoolean(closure.call(it)) - if( result ) target.bind(it) - }) - } - - return target + return new FilterOp() + .withSource(source) + .withCriteria(closure) + .apply() } DataflowWriteChannel until(DataflowReadChannel source, final Closure closure) { return new UntilOp(source,closure).apply() } - /** * Modifies this collection to remove all duplicated items, using the default comparator. * diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/ReduceOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/ReduceOp.groovy index 7a740e068b..881638f328 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/ReduceOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/ReduceOp.groovy @@ -26,6 +26,8 @@ import groovyx.gpars.dataflow.operator.DataflowProcessor import nextflow.Channel import nextflow.Global import nextflow.Session +import nextflow.extension.op.Op + /** * Implements reduce operator logic * @@ -117,7 +119,7 @@ class ReduceOp { final result = beforeBind ? beforeBind.call(accum) : accum - Op.bind(target, result) + Op.bind(processor, target, result) } boolean onException(final DataflowProcessor processor, final Throwable e) { diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/ToListOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/ToListOp.groovy index 16bdb74ab1..2247145271 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/ToListOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/ToListOp.groovy @@ -20,6 +20,8 @@ import groovy.transform.CompileStatic import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowVariable import groovyx.gpars.dataflow.expression.DataflowExpression +import groovyx.gpars.dataflow.operator.DataflowProcessor +import nextflow.extension.op.Op /** * Implements {@link OperatorImpl#toList(groovyx.gpars.dataflow.DataflowReadChannel)} operator @@ -52,7 +54,7 @@ class ToListOp { final result = new ArrayList(1) Map events = [:] events.onNext = { result.add(it) } - events.onComplete = { Op.bind(target, result) } + events.onComplete = { DataflowProcessor processor -> Op.bind(processor, target, result) } DataflowHelper.subscribeImpl(source, events) return target } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy new file mode 100644 index 0000000000..f180d57212 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy @@ -0,0 +1,83 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.extension.op + +import java.util.concurrent.ConcurrentHashMap + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import groovyx.gpars.dataflow.DataflowWriteChannel +import groovyx.gpars.dataflow.operator.DataflowProcessor +import groovyx.gpars.dataflow.operator.PoisonPill +import nextflow.Global +import nextflow.Session +import nextflow.prov.Prov +import nextflow.prov.Tracker +/** + * Operator helpers methods + * + * @author Paolo Di Tommaso + */ +@Slf4j +@CompileStatic +class Op { + + static final public ConcurrentHashMap context = new ConcurrentHashMap<>(); + + static List unwrap(List messages) { + final ArrayList result = new ArrayList<>(); + for( Object it : messages ) { + result.add(it instanceof Tracker.Msg ? ((Tracker.Msg)it).value : it); + } + return result; + } + + static Object unwrap(Object it) { + return it instanceof Tracker.Msg ? it.value : it + } + + static Tracker.Msg wrap(Object obj) { + obj instanceof Tracker.Msg ? obj : Tracker.Msg.of(obj) + } + + static void bind(DataflowProcessor operator, DataflowWriteChannel channel, Object msg) { + try { + if( msg instanceof PoisonPill ) { + channel.bind(msg); + context.remove(operator); + } + else { + final ctx = context.get(operator) + if( !ctx ) + throw new IllegalStateException("Cannot find any context for operator=$operator") + Prov.getTracker().bindOutput(ctx.getPreviousRun(), channel, msg) + } + } + catch (Throwable t) { + log.error("Unexpected resolving execution provenance: ${t.message}", t) + (Global.session as Session).abort(t) + } + } + + static OpClosure instrument(Closure op, boolean accumulator=false) { + return accumulator + ? new OpAccumulatorClosure(op) + : new OpClosure(op) + } + +} diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/op/OpAccumulatorClosure.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/op/OpAccumulatorClosure.groovy new file mode 100644 index 0000000000..cbb29a4004 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/extension/op/OpAccumulatorClosure.groovy @@ -0,0 +1,42 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.extension.op + +import nextflow.prov.OperatorRun + +/** + * A closure that wraps the execution of an operator target code (closure) + * and maps the inputs and outputs to the corresponding operator run. + * + * This class extends {@link OpClosure} assuming that all results are `"accumulated"` + * as they were executed in the same operator run + * + * @author Paolo Di Tommaso + */ +class OpAccumulatorClosure extends OpClosure { + + OpAccumulatorClosure(Closure code) { + super(code) + setPreviousRun(new OperatorRun()) + } + + @Override + protected OperatorRun runInstance() { + return getPreviousRun() + } +} diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/op/OpClosure.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/op/OpClosure.groovy new file mode 100644 index 0000000000..55f9f53f9c --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/extension/op/OpClosure.groovy @@ -0,0 +1,126 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.extension.op + +import java.util.concurrent.ConcurrentHashMap + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.prov.OperatorRun +import nextflow.prov.Prov +import org.codehaus.groovy.runtime.InvokerHelper +/** + * A closure that wraps the execution of an operator target code (closure) + * and maps the inputs and outputs to the corresponding operator run. + * + * @author Paolo Di Tommaso + */ +@Slf4j +@CompileStatic +class OpClosure extends Closure implements OpContext { + + private final Closure target + + private final ThreadLocal runPerThread = ThreadLocal.withInitial(()->new OperatorRun()) + + private final Map holder = new ConcurrentHashMap<>() + + @Override + OperatorRun getPreviousRun() { + return holder.get('previousRun') + } + + protected void setPreviousRun(OperatorRun run) { + holder.put('previousRun', run) + } + + OpClosure(Closure code) { + super(code.getOwner(), code.getThisObject()) + this.target = code + this.target.setDelegate(code.getDelegate()) + this.target.setResolveStrategy(code.getResolveStrategy()) + } + + @Override + Class[] getParameterTypes() { + return target.getParameterTypes() + } + + @Override + int getMaximumNumberOfParameters() { + return target.getMaximumNumberOfParameters() + } + + @Override + Object getDelegate() { + return target.getDelegate() + } + + @Override + int getDirective() { + return target.getDirective() + } + + @Override + void setDelegate(Object delegate) { + target.setDelegate(delegate) + } + + @Override + void setDirective(int directive) { + target.setDirective(directive) + } + + @Override + void setResolveStrategy(int resolveStrategy) { + target.setResolveStrategy(resolveStrategy) + } + + @Override + void setProperty(String propertyName, Object newValue) { + target.setProperty(propertyName, newValue) + } + + protected OperatorRun runInstance() { + final result = runPerThread.get() + setPreviousRun(result) + return result + } + + @Override + Object call(final Object... args) { + // when the accumulator flag true, re-use the previous run object + final OperatorRun run = runInstance() + // map the inputs + final List inputs = Prov.getTracker().receiveInputs(run, Arrays.asList(args)) + final Object result = InvokerHelper.invokeMethod(target, "call", inputs.toArray()) + // return the operation result + return result + } + + @Override + Object call(Object args) { + return call(InvokerHelper.asArray(args)) + } + + @Override + Object call() { + return call(InvokerHelper.EMPTY_ARGS) + } + +} diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/op/OpContext.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/op/OpContext.groovy new file mode 100644 index 0000000000..3fd775c068 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/extension/op/OpContext.groovy @@ -0,0 +1,29 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.extension.op + +import nextflow.prov.OperatorRun + +/** + * Model an operator run context + * + * @author Paolo Di Tommaso + */ +interface OpContext { + OperatorRun getPreviousRun() +} diff --git a/modules/nextflow/src/main/groovy/nextflow/prov/Tracker.groovy b/modules/nextflow/src/main/groovy/nextflow/prov/Tracker.groovy index 8d9a6329a3..db21d2bbb7 100644 --- a/modules/nextflow/src/main/groovy/nextflow/prov/Tracker.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/prov/Tracker.groovy @@ -23,11 +23,12 @@ import groovy.transform.Canonical import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import groovyx.gpars.dataflow.DataflowWriteChannel -import nextflow.extension.Op +import nextflow.extension.op.Op import nextflow.processor.TaskId import nextflow.processor.TaskRun /** - * + * Implement provenance tracking logic for tasks and operators + * * @author Paolo Di Tommaso */ @Slf4j @@ -143,20 +144,22 @@ class Tracker { } private void logOutput(TrailRun run, Msg msg) { - String str - if( run instanceof OperatorRun ) { - str = "Operator output" - str += "\n - id : ${System.identityHashCode(run)}" - } - else if( run instanceof TaskRun ) { - str = "Task output" - str += "\n - id : ${run.id}" - str += "\n - name: '${run.name}'" + if( log.isTraceEnabled() ) { + String str + if( run instanceof OperatorRun ) { + str = "Operator output" + str += "\n - id : ${System.identityHashCode(run)}" + } + else if( run instanceof TaskRun ) { + str = "Task output" + str += "\n - id : ${run.id}" + str += "\n - name: '${run.name}'" + } + else + throw new IllegalArgumentException("Unknown run type: ${run}") + str += "\n=> ${msg}" + log.trace(str) } - else - throw new IllegalArgumentException("Unknown run type: ${run}") - str += "\n=> ${msg}" - log.trace(str) } } diff --git a/modules/nextflow/src/main/groovy/nextflow/prov/TrailRun.groovy b/modules/nextflow/src/main/groovy/nextflow/prov/TrailRun.groovy index 0f04c9e6d9..32e1eb4b95 100644 --- a/modules/nextflow/src/main/groovy/nextflow/prov/TrailRun.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/prov/TrailRun.groovy @@ -18,6 +18,7 @@ package nextflow.prov /** + * Marker interface to identity a {@link nextflow.processor.TaskRun} or {@link OperatorRun} * * @author Paolo Di Tommaso */ diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/OpTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/OpTest.groovy index e55decaa7d..dff1f10c29 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/OpTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/OpTest.groovy @@ -17,10 +17,8 @@ package nextflow.extension - -import nextflow.prov.Prov +import nextflow.extension.op.Op import spock.lang.Specification - /** * * @author Paolo Di Tommaso @@ -38,10 +36,6 @@ class OpTest extends Specification { def z = c.call([v1, v2] as Object[]) then: z == 3 - and: - Op.currentOperator.get().inputIds == [ System.identityHashCode(v1), System.identityHashCode(v2) ] - cleanup: - Prov.clear() } } diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/op/OpClosureTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/op/OpClosureTest.groovy new file mode 100644 index 0000000000..277eb55d8f --- /dev/null +++ b/modules/nextflow/src/test/groovy/nextflow/extension/op/OpClosureTest.groovy @@ -0,0 +1,41 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.extension.op + +import spock.lang.Specification + +/** + * + * @author Paolo Di Tommaso + */ +class OpClosureTest extends Specification { + + def 'should invoke target closure' () { + given: + def code = { a,b -> a+b } + def wrapper = new OpClosure(code) + + when: + def result = wrapper.call(1,2) + then: + result == 3 + and: + wrapper.getPreviousRun() != null + } + +} diff --git a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy index f13bd7a7b2..1330a45b97 100644 --- a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy @@ -258,4 +258,31 @@ class ProvTest extends Dsl2Spec { } + def 'should track provenance two processes and the filter operator'() { + + when: + dsl_eval(globalConfig(), ''' + workflow { + channel.of(1,2,3) | p1 | filter { x-> x<30 } | p2 + } + + process p1 { + input: val(x) + output: val(y) + exec: + y = x*10 + } + + process p2 { + input: val(x) + exec: + println x + } + ''') + + then: + def upstream = upstreamTasksOf('p2 (1)') + upstream.size() == 1 + upstream.first.name == 'p1 (1)' + } } From f06256fd1bd7814d7c336172eb08422ee20d7581 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Thu, 9 Jan 2025 15:18:12 +0100 Subject: [PATCH 14/66] Fix OperatorRun allocation Signed-off-by: Paolo Di Tommaso --- .../nextflow/extension/op/OpAccumulatorClosure.groovy | 2 +- .../main/groovy/nextflow/extension/op/OpClosure.groovy | 9 +++++---- .../src/test/groovy/nextflow/prov/ProvTest.groovy | 4 ++-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/op/OpAccumulatorClosure.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/op/OpAccumulatorClosure.groovy index cbb29a4004..d60306dc89 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/op/OpAccumulatorClosure.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/op/OpAccumulatorClosure.groovy @@ -36,7 +36,7 @@ class OpAccumulatorClosure extends OpClosure { } @Override - protected OperatorRun runInstance() { + protected OperatorRun allocateRun() { return getPreviousRun() } } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/op/OpClosure.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/op/OpClosure.groovy index 55f9f53f9c..efca7f304d 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/op/OpClosure.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/op/OpClosure.groovy @@ -36,7 +36,7 @@ class OpClosure extends Closure implements OpContext { private final Closure target - private final ThreadLocal runPerThread = ThreadLocal.withInitial(()->new OperatorRun()) + private final ThreadLocal runPerThread = new ThreadLocal<>() private final Map holder = new ConcurrentHashMap<>() @@ -96,8 +96,9 @@ class OpClosure extends Closure implements OpContext { target.setProperty(propertyName, newValue) } - protected OperatorRun runInstance() { - final result = runPerThread.get() + protected OperatorRun allocateRun() { + final result = new OperatorRun() + runPerThread.set(result) setPreviousRun(result) return result } @@ -105,7 +106,7 @@ class OpClosure extends Closure implements OpContext { @Override Object call(final Object... args) { // when the accumulator flag true, re-use the previous run object - final OperatorRun run = runInstance() + final OperatorRun run = allocateRun() // map the inputs final List inputs = Prov.getTracker().receiveInputs(run, Arrays.asList(args)) final Object result = InvokerHelper.invokeMethod(target, "call", inputs.toArray()) diff --git a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy index 1330a45b97..2bc9be4c25 100644 --- a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy @@ -51,7 +51,7 @@ class ProvTest extends Dsl2Spec { upstream.first.name == 'p1' } - def 'should branch two process'() { + def 'should track provenance with branch operator'() { when: dsl_eval(globalConfig(), ''' @@ -258,7 +258,7 @@ class ProvTest extends Dsl2Spec { } - def 'should track provenance two processes and the filter operator'() { + def 'should track provenance with filter operator'() { when: dsl_eval(globalConfig(), ''' From 7848791e9623c139b1883de883ca2105c81b5ef6 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Thu, 9 Jan 2025 17:15:11 +0100 Subject: [PATCH 15/66] Add distinc + unique operators Signed-off-by: Paolo Di Tommaso --- .../nextflow/extension/DistinctOp.groovy | 79 +++++++++++++++ .../nextflow/extension/OperatorImpl.groovy | 62 ++---------- .../groovy/nextflow/extension/ReduceOp.groovy | 2 +- .../groovy/nextflow/extension/UniqueOp.groovy | 99 +++++++++++++++++++ .../test/groovy/nextflow/prov/ProvTest.groovy | 89 ++++++++++++++++- 5 files changed, 272 insertions(+), 59 deletions(-) create mode 100644 modules/nextflow/src/main/groovy/nextflow/extension/DistinctOp.groovy create mode 100644 modules/nextflow/src/main/groovy/nextflow/extension/UniqueOp.groovy diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/DistinctOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/DistinctOp.groovy new file mode 100644 index 0000000000..c2bc574602 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/extension/DistinctOp.groovy @@ -0,0 +1,79 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.extension + +import groovyx.gpars.dataflow.DataflowReadChannel +import groovyx.gpars.dataflow.DataflowWriteChannel +import groovyx.gpars.dataflow.operator.DataflowProcessor +import nextflow.extension.op.Op +/** + * Implements the "distinct" operator logic + * + * @author Paolo Di Tommaso + */ +class DistinctOp { + + private DataflowReadChannel source + private DataflowWriteChannel target + private Closure comparator + + DistinctOp() {} + + DistinctOp withSource(DataflowReadChannel source) { + assert source!=null + this.source = source + return this + } + + DistinctOp withTarget(DataflowWriteChannel target) { + assert target!=null + this.target = target + return this + } + + DistinctOp withComparator(Closure comparator) { + assert comparator!=null + this.comparator = comparator + return this + } + + DataflowWriteChannel apply() { + assert source != null + assert comparator != null + + if( !target ) + target = CH.createBy(source) + + final params = new DataflowHelper.OpParams() + .withInput(source) + .withOutput(target) + + def previous = null + DataflowHelper.newOperator(params) { + final proc = ((DataflowProcessor) getDelegate()) + def key = comparator.call(it) + if( key != previous ) { + previous = key + Op.bind(proc, target, it) + } + } + + return target + } + +} diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy index 686d0462d1..cf39c74084 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy @@ -291,43 +291,11 @@ class OperatorImpl { * @param comparator * @return */ - DataflowWriteChannel unique(final DataflowReadChannel source, Closure comparator ) { - - final history = [:] - final target = CH.createBy(source) - final stopOnFirst = source instanceof DataflowExpression - - // when the operator stop clear the history map - final events = new DataflowEventAdapter() { - void afterStop(final DataflowProcessor processor) { - history.clear() - } - } - - final filter = { - try { - final key = comparator.call(it) - if( history.containsKey(key) ) { - return Channel.VOID - } - else { - history.put(key,true) - return it - } - } - finally { - if( stopOnFirst ) { - ((DataflowProcessor) getDelegate()).terminate() - } - } - } as Closure - - return ChainOp.create() - .withSource(source) - .withTarget(target) - .withListener(events) - .withAction(filter) - .apply() + DataflowWriteChannel unique(final DataflowReadChannel source, final Closure comparator) { + return new UniqueOp() + .withSource(source) + .withComparator(comparator) + .apply() } /** @@ -349,24 +317,10 @@ class OperatorImpl { * * @return */ - DataflowWriteChannel distinct( final DataflowReadChannel source, Closure comparator ) { - - def previous = null - final target = CH.createBy(source) - Closure filter = { it -> - - def key = comparator.call(it) - if( key == previous ) { - return Channel.VOID - } - previous = key - return it - } - - return ChainOp.create() + DataflowWriteChannel distinct(final DataflowReadChannel source, final Closure comparator) { + new DistinctOp() .withSource(source) - .withTarget(target) - .withAction(filter) + .withComparator(comparator) .apply() } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/ReduceOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/ReduceOp.groovy index 881638f328..b4a3000eef 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/ReduceOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/ReduceOp.groovy @@ -93,7 +93,7 @@ class ReduceOp { def accum = this.seed // intercepts operator events - def listener = new DataflowEventAdapter() { + final listener = new DataflowEventAdapter() { /* * call the passed closure each time */ diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/UniqueOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/UniqueOp.groovy new file mode 100644 index 0000000000..f5a4436182 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/extension/UniqueOp.groovy @@ -0,0 +1,99 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.extension + +import groovyx.gpars.dataflow.DataflowReadChannel +import groovyx.gpars.dataflow.DataflowWriteChannel +import groovyx.gpars.dataflow.expression.DataflowExpression +import groovyx.gpars.dataflow.operator.DataflowEventAdapter +import groovyx.gpars.dataflow.operator.DataflowProcessor +import nextflow.extension.op.Op + +/** + * Implements the "unique" operator logic + * + * @author Paolo Di Tommaso + */ +class UniqueOp { + + private DataflowReadChannel source + private DataflowWriteChannel target + private Closure comparator + + UniqueOp() {} + + UniqueOp withSource(DataflowReadChannel source) { + assert source!=null + this.source = source + return this + } + + UniqueOp withTarget(DataflowWriteChannel target) { + assert target!=null + this.target = target + return this + } + + UniqueOp withComparator(Closure comparator) { + assert comparator!=null + this.comparator = comparator + return this + } + + DataflowWriteChannel apply() { + assert source != null + assert comparator != null + + if( !target ) + target = CH.createBy(source) + + final history = new LinkedHashMap() + final stopOnFirst = source instanceof DataflowExpression + + // when the operator stop clear the history map + final listener = new DataflowEventAdapter() { + void afterStop(final DataflowProcessor processor) { + history.clear() + } + } + + final params = new DataflowHelper.OpParams() + .withInput(source) + .withOutput(target) + .withListener(listener) + + DataflowHelper.newOperator(params) { + final proc = ((DataflowProcessor) getDelegate()) + try { + final key = comparator.call(it) + if( !history.containsKey(key) ) { + history.put(key,true) + Op.bind(proc, target, it) + } + } + finally { + if( stopOnFirst ) { + proc.terminate() + } + } + } + + return target + } + +} diff --git a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy index 2bc9be4c25..055541d70e 100644 --- a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy @@ -263,7 +263,7 @@ class ProvTest extends Dsl2Spec { when: dsl_eval(globalConfig(), ''' workflow { - channel.of(1,2,3) | p1 | filter { x-> x<30 } | p2 + channel.of(1,3,2) | p1 | filter { x-> x<30 } | p2 } process p1 { @@ -281,8 +281,89 @@ class ProvTest extends Dsl2Spec { ''') then: - def upstream = upstreamTasksOf('p2 (1)') - upstream.size() == 1 - upstream.first.name == 'p1 (1)' + def upstream1 = upstreamTasksOf('p2 (1)') + upstream1.size() == 1 + upstream1.first.name == 'p1 (1)' + then: + def upstream2 = upstreamTasksOf('p2 (2)') + upstream2.size() == 1 + upstream2.first.name == 'p1 (3)' + } + + def 'should track provenance with unique operator'() { + + when: + dsl_eval(globalConfig(), ''' + workflow { + channel.of(1,2,2,3) | p1 | unique | p2 + } + + process p1 { + input: val(x) + output: val(y) + exec: + y = x + } + + process p2 { + input: val(x) + exec: + println x + } + ''') + + then: + def upstream1 = upstreamTasksOf('p2 (1)') + upstream1.size() == 1 + upstream1.first.name == 'p1 (1)' + + then: + def upstream2 = upstreamTasksOf('p2 (2)') + upstream2.size() == 1 + upstream2.first.name == 'p1 (2)' + + then: + def upstream3 = upstreamTasksOf('p2 (3)') + upstream3.size() == 1 + upstream3.first.name == 'p1 (4)' + } + + + def 'should track provenance with distinc operator'() { + + when: + dsl_eval(globalConfig(), ''' + workflow { + channel.of(1,2,2,2,3) | p1 | unique | p2 + } + + process p1 { + input: val(x) + output: val(y) + exec: + y = x + } + + process p2 { + input: val(x) + exec: + println x + } + ''') + + then: + def upstream1 = upstreamTasksOf('p2 (1)') + upstream1.size() == 1 + upstream1.first.name == 'p1 (1)' + + then: + def upstream2 = upstreamTasksOf('p2 (2)') + upstream2.size() == 1 + upstream2.first.name == 'p1 (2)' + + then: + def upstream3 = upstreamTasksOf('p2 (3)') + upstream3.size() == 1 + upstream3.first.name == 'p1 (5)' } } From 0014db9f8f224967f15ea70e26544b18dfb97a06 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Thu, 9 Jan 2025 18:34:15 +0100 Subject: [PATCH 16/66] Add first operator Signed-off-by: Paolo Di Tommaso --- .../nextflow/extension/DistinctOp.groovy | 6 +- .../groovy/nextflow/extension/FilterOp.groovy | 7 +- .../groovy/nextflow/extension/FirstOp.groovy | 94 +++++++++++++++++++ .../groovy/nextflow/extension/MapOp.groovy | 4 +- .../nextflow/extension/OperatorImpl.groovy | 31 ++---- .../groovy/nextflow/extension/UniqueOp.groovy | 4 +- .../groovy/nextflow/extension/UntilOp.groovy | 4 +- .../extension/OperatorDsl2Test.groovy | 25 ++--- .../test/groovy/nextflow/prov/ProvTest.groovy | 28 ++++++ 9 files changed, 159 insertions(+), 44 deletions(-) create mode 100644 modules/nextflow/src/main/groovy/nextflow/extension/FirstOp.groovy diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/DistinctOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/DistinctOp.groovy index c2bc574602..42be8a6e0c 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/DistinctOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/DistinctOp.groovy @@ -17,6 +17,7 @@ package nextflow.extension +import groovy.transform.CompileStatic import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowWriteChannel import groovyx.gpars.dataflow.operator.DataflowProcessor @@ -26,6 +27,7 @@ import nextflow.extension.op.Op * * @author Paolo Di Tommaso */ +@CompileStatic class DistinctOp { private DataflowReadChannel source @@ -65,8 +67,8 @@ class DistinctOp { def previous = null DataflowHelper.newOperator(params) { - final proc = ((DataflowProcessor) getDelegate()) - def key = comparator.call(it) + final proc = getDelegate() as DataflowProcessor + final key = comparator.call(it) if( key != previous ) { previous = key Op.bind(proc, target, it) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/FilterOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/FilterOp.groovy index 3e38e82187..e3250216bc 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/FilterOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/FilterOp.groovy @@ -19,6 +19,7 @@ package nextflow.extension import static nextflow.extension.DataflowHelper.* +import groovy.transform.CompileStatic import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowWriteChannel import groovyx.gpars.dataflow.expression.DataflowExpression @@ -28,9 +29,11 @@ import nextflow.extension.op.Op import org.codehaus.groovy.runtime.callsite.BooleanReturningMethodInvoker import org.codehaus.groovy.runtime.typehandling.DefaultTypeTransformation /** - * + * Implements the "filter" operator logic + * * @author Paolo Di Tommaso */ +@CompileStatic class FilterOp { private DataflowReadChannel source @@ -65,7 +68,7 @@ class FilterOp { final result = criteria instanceof Closure ? DefaultTypeTransformation.castToBoolean(criteria.call(it)) : discriminator.invoke(criteria, (Object)it) - final proc = ((DataflowProcessor) getDelegate()) + final proc = getDelegate() as DataflowProcessor if( result ) { Op.bind(proc, target, it) } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/FirstOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/FirstOp.groovy new file mode 100644 index 0000000000..4869596cee --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/extension/FirstOp.groovy @@ -0,0 +1,94 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.extension + +import static nextflow.extension.DataflowHelper.* + +import groovy.transform.CompileStatic +import groovyx.gpars.dataflow.DataflowReadChannel +import groovyx.gpars.dataflow.DataflowVariable +import groovyx.gpars.dataflow.DataflowWriteChannel +import groovyx.gpars.dataflow.expression.DataflowExpression +import groovyx.gpars.dataflow.operator.DataflowEventAdapter +import groovyx.gpars.dataflow.operator.DataflowProcessor +import nextflow.Channel +import nextflow.extension.op.Op +import org.codehaus.groovy.runtime.callsite.BooleanReturningMethodInvoker +/** + * + * @author Paolo Di Tommaso + */ +@CompileStatic +class FirstOp { + + private DataflowReadChannel source + private DataflowVariable target + private Object criteria + + FirstOp() {} + + FirstOp withSource(DataflowReadChannel source) { + assert source!=null + this.source = source + return this + } + + FirstOp withTarget(DataflowVariable target) { + assert target!=null + this.target = target + return this + } + + FirstOp withCriteria(Object criteria) { + assert criteria!=null + this.criteria = criteria + return this + } + + DataflowWriteChannel apply() { + assert source!=null + assert criteria!=null + + if( target==null ) + target = new DataflowVariable() + + final stopOnFirst = source instanceof DataflowExpression + final discriminator = new BooleanReturningMethodInvoker("isCase"); + + final params = new OpParams() + .withInput(source) + .withListener(new DataflowEventAdapter() { + @Override + void afterStop(DataflowProcessor proc) { + if( stopOnFirst && !target.isBound() ) + Op.bind(proc, target, Channel.STOP) + } + }) + + newOperator(params) { + final proc = getDelegate() as DataflowProcessor + final accept = discriminator.invoke(criteria, it) + if( accept ) + Op.bind(proc, target, it) + if( accept || stopOnFirst ) + proc.terminate() + } + + return target + } +} diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/MapOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/MapOp.groovy index 78dafff250..735f8aba58 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/MapOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/MapOp.groovy @@ -16,6 +16,7 @@ package nextflow.extension +import groovy.transform.CompileStatic import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowWriteChannel import groovyx.gpars.dataflow.expression.DataflowExpression @@ -28,6 +29,7 @@ import nextflow.extension.op.Op * * @author Paolo Di Tommaso */ +@CompileStatic class MapOp { private DataflowReadChannel source @@ -55,7 +57,7 @@ class MapOp { DataflowHelper.newOperator(source, target) { it -> final result = mapper.call(it) - final proc = (DataflowProcessor) getDelegate() + final proc = getDelegate() as DataflowProcessor // bind the result value if (result != Channel.VOID) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy index cf39c74084..5eff3cd920 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy @@ -28,7 +28,6 @@ import groovyx.gpars.dataflow.DataflowVariable import groovyx.gpars.dataflow.DataflowWriteChannel import groovyx.gpars.dataflow.expression.DataflowExpression import groovyx.gpars.dataflow.operator.ChainWithClosure -import groovyx.gpars.dataflow.operator.ControlMessage import groovyx.gpars.dataflow.operator.DataflowEventAdapter import groovyx.gpars.dataflow.operator.DataflowProcessor import groovyx.gpars.dataflow.operator.PoisonPill @@ -342,9 +341,7 @@ class OperatorImpl { log.warn msg } - def target = new DataflowVariable() - source.whenBound { target.bind(it) } - return target + return first(source, { true }) } /** @@ -356,27 +353,11 @@ class OperatorImpl { * @param source * @return */ - DataflowWriteChannel first( final DataflowReadChannel source, Object criteria ) { - - def target = new DataflowVariable() - def discriminator = new BooleanReturningMethodInvoker("isCase"); - - if( source instanceof DataflowExpression ) { - source.whenBound { - def result = it instanceof ControlMessage ? false : discriminator.invoke(criteria, it) - target.bind( result ? it : Channel.STOP ) - } - } - else { - newOperator([source],[]) { - if( discriminator.invoke(criteria, it) ) { - target.bind(it) - ((DataflowProcessor) getDelegate()).terminate() - } - } - } - - return target + DataflowWriteChannel first( final DataflowReadChannel source, final Object criteria ) { + new FirstOp() + .withSource(source) + .withCriteria(criteria) + .apply() } /** diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/UniqueOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/UniqueOp.groovy index f5a4436182..8413539abc 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/UniqueOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/UniqueOp.groovy @@ -17,6 +17,7 @@ package nextflow.extension +import groovy.transform.CompileStatic import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowWriteChannel import groovyx.gpars.dataflow.expression.DataflowExpression @@ -29,6 +30,7 @@ import nextflow.extension.op.Op * * @author Paolo Di Tommaso */ +@CompileStatic class UniqueOp { private DataflowReadChannel source @@ -78,7 +80,7 @@ class UniqueOp { .withListener(listener) DataflowHelper.newOperator(params) { - final proc = ((DataflowProcessor) getDelegate()) + final proc = getDelegate() as DataflowProcessor try { final key = comparator.call(it) if( !history.containsKey(key) ) { diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/UntilOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/UntilOp.groovy index cb9ebefc6b..99a8670cf8 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/UntilOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/UntilOp.groovy @@ -19,6 +19,7 @@ package nextflow.extension import static nextflow.extension.DataflowHelper.* +import groovy.transform.CompileStatic import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowWriteChannel import groovyx.gpars.dataflow.operator.DataflowProcessor @@ -29,6 +30,7 @@ import org.codehaus.groovy.runtime.typehandling.DefaultTypeTransformation * * @author Paolo Di Tommaso */ +@CompileStatic class UntilOp { private DataflowReadChannel source @@ -44,7 +46,7 @@ class UntilOp { newOperator(source, target, { final result = DefaultTypeTransformation.castToBoolean(closure.call(it)) - final proc = ((DataflowProcessor) getDelegate()) + final proc = getDelegate() as DataflowProcessor if( result ) { proc.bindOutput(Channel.STOP) diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/OperatorDsl2Test.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/OperatorDsl2Test.groovy index 47caf5abe6..82af39dca7 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/OperatorDsl2Test.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/OperatorDsl2Test.groovy @@ -17,6 +17,7 @@ package nextflow.extension +import groovyx.gpars.dataflow.DataflowReadChannel import nextflow.Channel import spock.lang.Timeout import test.Dsl2Spec @@ -30,35 +31,35 @@ class OperatorDsl2Test extends Dsl2Spec { def 'should test unique' () { when: - def channel = dsl_eval(""" + def result = dsl_eval(""" Channel.of(1,2,3).unique() - """) + """) as DataflowReadChannel then: - channel.val == 1 - channel.val == 2 - channel.val == 3 - channel.val == Channel.STOP + result.unwrap() == 1 + result.unwrap() == 2 + result.unwrap() == 3 + result.unwrap() == Channel.STOP } def 'should test unique with value' () { when: - def channel = dsl_eval(""" + def result = dsl_eval(""" Channel.value(1).unique() - """) + """) as DataflowReadChannel then: - channel.val == 1 + result.unwrap() == 1 } def 'should test unique with collect' () { when: - def ch = dsl_eval(""" + def result = dsl_eval(""" Channel.of( 'a', 'b', 'c') .collect() .unique() .view() - """) + """) as DataflowReadChannel then: - ch.val == ['a','b','c'] + result.unwrap() == ['a','b','c'] } } diff --git a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy index 055541d70e..6a6ab327e9 100644 --- a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy @@ -366,4 +366,32 @@ class ProvTest extends Dsl2Spec { upstream3.size() == 1 upstream3.first.name == 'p1 (5)' } + + def 'should track provenance with first operator'() { + + when: + dsl_eval(globalConfig(), ''' + workflow { + channel.of(1,2,3) | p1 | first | p2 + } + + process p1 { + input: val(x) + output: val(y) + exec: + y = x + } + + process p2 { + input: val(x) + exec: + println x + } + ''') + + then: + def upstream1 = upstreamTasksOf('p2') + upstream1.size() == 1 + upstream1.first.name == 'p1 (1)' + } } From 2b4c5b3e3d62db87d682dabbdabc224c6b859a51 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Thu, 9 Jan 2025 18:58:29 +0100 Subject: [PATCH 17/66] Add take and last operator Signed-off-by: Paolo Di Tommaso --- .../groovy/nextflow/extension/LastOp.groovy | 62 +++++++++++++++++++ .../nextflow/extension/OperatorImpl.groovy | 7 +-- .../groovy/nextflow/extension/TakeOp.groovy | 19 +++--- .../test/groovy/nextflow/prov/ProvTest.groovy | 61 +++++++++++++++++- 4 files changed, 135 insertions(+), 14 deletions(-) create mode 100644 modules/nextflow/src/main/groovy/nextflow/extension/LastOp.groovy diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/LastOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/LastOp.groovy new file mode 100644 index 0000000000..28e3d487f6 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/extension/LastOp.groovy @@ -0,0 +1,62 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.extension + + +import groovy.transform.CompileStatic +import groovyx.gpars.dataflow.DataflowReadChannel +import groovyx.gpars.dataflow.DataflowVariable +import groovyx.gpars.dataflow.operator.DataflowProcessor +import nextflow.extension.op.Op + +/** + * Implements last operator + * + * @author Paolo Di Tommaso + */ +@CompileStatic +class LastOp { + + private DataflowReadChannel source + private DataflowVariable target + + LastOp withSource(DataflowReadChannel source) { + assert source!=null + this.source = source + return this + } + + LastOp withTarget(DataflowVariable target) { + assert target!=null + this.target = target + return this + } + + DataflowVariable apply() { + assert source!=null + if( target==null ) + target = new DataflowVariable() + + def last = null + final next = { last = it } + final done = { DataflowProcessor proc -> Op.bind(proc, target, last) } + DataflowHelper.subscribeImpl( source, [onNext:next, onComplete: done] ) + return target + } + +} diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy index 5eff3cd920..4a53489ab5 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy @@ -384,11 +384,7 @@ class OperatorImpl { * @return A {@code DataflowVariable} emitting the `last` item in the channel */ DataflowWriteChannel last( final DataflowReadChannel source ) { - - def target = new DataflowVariable() - def last = null - subscribeImpl( source, [onNext: { last = it }, onComplete: { target.bind(last) }] ) - return target + new LastOp().withSource(source).apply() } DataflowWriteChannel collect(final DataflowReadChannel source, Closure action=null) { @@ -399,7 +395,6 @@ class OperatorImpl { return new CollectOp(source,action,opts).apply() } - /** * Convert a {@code DataflowQueue} alias *channel* to a Java {@code List} * diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/TakeOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/TakeOp.groovy index f7bf259d0a..b7bd3a7c6e 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/TakeOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/TakeOp.groovy @@ -23,15 +23,15 @@ import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowWriteChannel -import groovyx.gpars.dataflow.operator.ChainWithClosure -import groovyx.gpars.dataflow.operator.CopyChannelsClosure import groovyx.gpars.dataflow.operator.DataflowEventAdapter import groovyx.gpars.dataflow.operator.DataflowProcessor import nextflow.Channel import nextflow.Global import nextflow.Session +import nextflow.extension.op.Op /** + * Implement "take" operator * * @author Paolo Di Tommaso */ @@ -73,11 +73,16 @@ class TakeOp { } } - newOperator( - inputs: [source], - outputs: [target], - listeners: (length > 0 ? [listener] : []), - new ChainWithClosure(new CopyChannelsClosure())) + final params = new OpParams() + .withInput(source) + .withOutput(target) + if( length>0 ) + params.withListener(listener) + + newOperator(params) { + final proc = getDelegate() as DataflowProcessor + Op.bind(proc, target, it) + } return target } diff --git a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy index 6a6ab327e9..c544aa2e3e 100644 --- a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy @@ -368,7 +368,6 @@ class ProvTest extends Dsl2Spec { } def 'should track provenance with first operator'() { - when: dsl_eval(globalConfig(), ''' workflow { @@ -394,4 +393,64 @@ class ProvTest extends Dsl2Spec { upstream1.size() == 1 upstream1.first.name == 'p1 (1)' } + + def 'should track provenance with take operator'() { + when: + dsl_eval(globalConfig(), ''' + workflow { + channel.of(1,2,3,4,5) | p1 | take(2) | p2 + } + + process p1 { + input: val(x) + output: val(y) + exec: + y = x + } + + process p2 { + input: val(x) + exec: + println x + } + ''') + + then: + def upstream1 = upstreamTasksOf('p2 (1)') + upstream1.size() == 1 + upstream1.first.name == 'p1 (1)' + then: + def upstream2 = upstreamTasksOf('p2 (2)') + upstream2.size() == 1 + upstream2.first.name == 'p1 (2)' + + } + + def 'should track provenance with last operator'() { + when: + dsl_eval(globalConfig(), ''' + workflow { + channel.of(1,2,3,4,5) | p1 | last | p2 + } + + process p1 { + input: val(x) + output: val(y) + exec: + y = x + } + + process p2 { + input: val(x) + exec: + println x + } + ''') + + then: + def upstream1 = upstreamTasksOf('p2') + upstream1.size() == 1 + upstream1.first.name == 'p1 (5)' + + } } From 17fced925b4f7c9e1d7218527bbc60db95008819 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Thu, 9 Jan 2025 19:49:57 +0100 Subject: [PATCH 18/66] Add count operator Signed-off-by: Paolo Di Tommaso --- .../nextflow/extension/CollectOp.groovy | 2 +- .../nextflow/extension/OperatorImpl.groovy | 33 ++++---- .../groovy/nextflow/extension/ReduceOp.groovy | 43 ++++------ .../test/groovy/nextflow/prov/ProvTest.groovy | 78 +++++++++++++++++++ 4 files changed, 109 insertions(+), 47 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/CollectOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/CollectOp.groovy index a75fc16fa4..af6fbeffc8 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/CollectOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/CollectOp.groovy @@ -62,7 +62,7 @@ class CollectOp { Op.bind(processor, target, msg) } - DataflowHelper.subscribeImpl(source, events) + DataflowHelper.subscribeImpl(source, true, events) return target } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy index 4a53489ab5..03139f7d64 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy @@ -91,6 +91,7 @@ class OperatorImpl { * @param closure * @return */ + @Deprecated DataflowWriteChannel chain(final DataflowReadChannel source, final Closure closure) { final target = CH.createBy(source) newOperator(source, target, stopErrorListener(source,target), new ChainWithClosure(closure)) @@ -104,6 +105,7 @@ class OperatorImpl { * @param closure * @return */ + @Deprecated DataflowWriteChannel chain(final DataflowReadChannel source, final Map params, final Closure closure) { return ChainOp.create() .withSource(source) @@ -423,8 +425,7 @@ class OperatorImpl { * @return */ DataflowWriteChannel count(final DataflowReadChannel source ) { - final target = count0(source, null) - return target + return count0(source, null) } /** @@ -435,8 +436,7 @@ class OperatorImpl { * @return */ DataflowWriteChannel count(final DataflowReadChannel source, final Object criteria ) { - final target = count0(source, criteria) - return target + return count0(source, criteria) } private static DataflowVariable count0(DataflowReadChannel source, Object criteria) { @@ -444,24 +444,17 @@ class OperatorImpl { final target = new DataflowVariable() final discriminator = criteria != null ? new BooleanReturningMethodInvoker("isCase") : null - if( source instanceof DataflowExpression) { - source.whenBound { item -> - discriminator == null || discriminator.invoke(criteria, item) ? target.bind(1) : target.bind(0) - } - } - else { - final action = { current, item -> - discriminator == null || discriminator.invoke(criteria, item) ? current+1 : current - } - - ReduceOp .create() - .withSource(source) - .withTarget(target) - .withSeed(0) - .withAction(action) - .apply() + final action = { current, item -> + discriminator == null || discriminator.invoke(criteria, item) ? current+1 : current } + ReduceOp .create() + .withSource(source) + .withTarget(target) + .withSeed(0) + .withAction(action) + .apply() + return target } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/ReduceOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/ReduceOp.groovy index b4a3000eef..11df7178bf 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/ReduceOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/ReduceOp.groovy @@ -17,17 +17,19 @@ package nextflow.extension +import static nextflow.extension.DataflowHelper.* + import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowVariable +import groovyx.gpars.dataflow.expression.DataflowExpression import groovyx.gpars.dataflow.operator.DataflowEventAdapter import groovyx.gpars.dataflow.operator.DataflowProcessor import nextflow.Channel import nextflow.Global import nextflow.Session import nextflow.extension.op.Op - /** * Implements reduce operator logic * @@ -88,30 +90,12 @@ class ReduceOp { throw new IllegalArgumentException("Missing reduce operator source channel") if( target==null ) target = new DataflowVariable() - + final stopOnFirst = source instanceof DataflowExpression // the *accumulator* value def accum = this.seed // intercepts operator events final listener = new DataflowEventAdapter() { - /* - * call the passed closure each time - */ - void afterRun(final DataflowProcessor processor, final List messages) { - final item = Op.unwrap(messages).get(0) - final value = accum == null ? item : action.call(accum, item) - - if( value == Channel.VOID ) { - // do nothing - } - else if( value == Channel.STOP ) { - processor.terminate() - } - else { - accum = value - } - } - /* * when terminates bind the result value */ @@ -129,13 +113,20 @@ class ReduceOp { } } - ChainOp.create() - .withSource(source) - .withTarget(CH.create()) - .withListener(listener) + final parameters = new OpParams() + .withInput(source) .withAccumulator(true) - .withAction({true}) - .apply() + .withListener(listener) + + newOperator(parameters) { + final value = accum == null ? it : action.call(accum, it) + final proc = getDelegate() as DataflowProcessor + if( value!=Channel.VOID && value!=Channel.STOP ) { + accum = value + } + if( stopOnFirst || value==Channel.STOP ) + proc.terminate() + } return target } diff --git a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy index c544aa2e3e..cde9819dd2 100644 --- a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy @@ -451,6 +451,84 @@ class ProvTest extends Dsl2Spec { def upstream1 = upstreamTasksOf('p2') upstream1.size() == 1 upstream1.first.name == 'p1 (5)' + } + + def 'should track provenance with collect operator'() { + when: + dsl_eval(globalConfig(), ''' + workflow { + channel.of(1,2,3) | p1 | collect | p2 + } + + process p1 { + input: val(x) + output: val(y) + exec: + y = x + } + + process p2 { + input: val(x) + exec: + println x + } + ''') + then: + def t1 = upstreamTasksOf('p2') + t1.name == ['p1 (1)', 'p1 (2)', 'p1 (3)'] } + + def 'should track provenance with count value operator'() { + when: + dsl_eval(globalConfig(), ''' + workflow { + channel.value(1) | p1 | count | p2 + } + + process p1 { + input: val(x) + output: val(y) + exec: + y = x + } + + process p2 { + input: val(x) + exec: + println x + } + ''') + + then: + def t1 = upstreamTasksOf('p2') + t1.name == ['p1'] + } + + def 'should track provenance with count many operator'() { + when: + dsl_eval(globalConfig(), ''' + workflow { + channel.of(1,2,3) | p1 | count | p2 + } + + process p1 { + input: val(x) + output: val(y) + exec: + y = x + } + + process p2 { + input: val(x) + exec: + println x + } + ''') + + then: + def t1 = upstreamTasksOf('p2') + t1.name == ['p1 (1)', 'p1 (2)', 'p1 (3)'] + } + } From e449538e82cbfbf96232b806c1db16a63b880470 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Fri, 10 Jan 2025 07:57:12 +0100 Subject: [PATCH 19/66] Add sum and mean ops Signed-off-by: Paolo Di Tommaso --- .../nextflow/extension/DistinctOp.groovy | 2 - .../groovy/nextflow/extension/MathOp.groovy | 135 ++++++++++++++++++ .../nextflow/extension/OperatorImpl.groovy | 63 ++------ 3 files changed, 145 insertions(+), 55 deletions(-) create mode 100644 modules/nextflow/src/main/groovy/nextflow/extension/MathOp.groovy diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/DistinctOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/DistinctOp.groovy index 42be8a6e0c..54b9eea32e 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/DistinctOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/DistinctOp.groovy @@ -34,8 +34,6 @@ class DistinctOp { private DataflowWriteChannel target private Closure comparator - DistinctOp() {} - DistinctOp withSource(DataflowReadChannel source) { assert source!=null this.source = source diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/MathOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/MathOp.groovy new file mode 100644 index 0000000000..d67587701e --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/extension/MathOp.groovy @@ -0,0 +1,135 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.extension + +import static nextflow.extension.DataflowHelper.* + +import groovy.transform.CompileDynamic +import groovy.transform.CompileStatic +import groovyx.gpars.dataflow.DataflowReadChannel +import groovyx.gpars.dataflow.DataflowVariable +import groovyx.gpars.dataflow.DataflowWriteChannel +import groovyx.gpars.dataflow.operator.DataflowProcessor +import nextflow.Channel +import nextflow.extension.op.Op +/** + * Implements the logic for "sum" and "mean" operators + * + * @author Paolo Di Tommaso + */ +@CompileStatic +class MathOp { + + private DataflowReadChannel source + private DataflowVariable target + private Closure action + private Aggregate aggregate + + private MathOp(Aggregate aggregate) { + this.aggregate = aggregate + } + + static MathOp sum() { + new MathOp(new Aggregate(name: 'sum')) + } + + static MathOp mean() { + new MathOp(new Aggregate(name: 'mean', mean: true)) + } + + MathOp withSource(DataflowReadChannel source) { + assert source!=null + this.source = source + return this + } + + MathOp withTarget(DataflowVariable target) { + assert target!=null + this.target = target + return this + } + + MathOp withAction(Closure action) { + this.action = action + this.aggregate.withAction(action) + return this + } + + DataflowWriteChannel apply() { + assert source!=null + if( target==null ) + target = new DataflowVariable() + subscribeImpl(source, [onNext: aggregate.&process, onComplete: this.&completion ]) + return target + } + + private void completion(DataflowProcessor proc) { + Op.bind(proc, target, aggregate.result ) + } + + /** + * Implements the logic for sum and mean operators + * + * @author Paolo Di Tommaso + */ + @CompileDynamic + static class Aggregate { + def accum + long count = 0 + boolean mean + Closure action + String name + + def process(it) { + if( it == null || it == Channel.VOID ) + return + + count++ + + def item = action != null ? action.call(it) : it + if( accum == null ) + accum = item + + else if( accum instanceof Number ) + accum += item + + else if( accum instanceof List && item instanceof List) + for( int i=0; i Date: Sat, 11 Jan 2025 17:51:06 +0100 Subject: [PATCH 20/66] Improve run operator allocation Signed-off-by: Paolo Di Tommaso --- .../nextflow/extension/DistinctOp.groovy | 4 +- .../nextflow/extension/OperatorImpl.groovy | 11 ++-- .../groovy/nextflow/extension/op/Op.groovy | 38 +++++++++----- ...losure.groovy => OpAbstractClosure.groovy} | 29 +++-------- .../nextflow/extension/op/OpContext.groovy | 2 +- ...losure.groovy => OpGroupingClosure.groovy} | 24 ++++++--- .../extension/op/OpRunningClosure.groovy | 52 +++++++++++++++++++ .../extension/op/OpClosureTest.groovy | 4 +- 8 files changed, 113 insertions(+), 51 deletions(-) rename modules/nextflow/src/main/groovy/nextflow/extension/op/{OpClosure.groovy => OpAbstractClosure.groovy} (80%) rename modules/nextflow/src/main/groovy/nextflow/extension/op/{OpAccumulatorClosure.groovy => OpGroupingClosure.groovy} (62%) create mode 100644 modules/nextflow/src/main/groovy/nextflow/extension/op/OpRunningClosure.groovy diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/DistinctOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/DistinctOp.groovy index 54b9eea32e..5552b4223e 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/DistinctOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/DistinctOp.groovy @@ -22,6 +22,8 @@ import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowWriteChannel import groovyx.gpars.dataflow.operator.DataflowProcessor import nextflow.extension.op.Op +import static nextflow.extension.DataflowHelper.newOperator + /** * Implements the "distinct" operator logic * @@ -64,7 +66,7 @@ class DistinctOp { .withOutput(target) def previous = null - DataflowHelper.newOperator(params) { + newOperator(params) { final proc = getDelegate() as DataflowProcessor final key = comparator.call(it) if( key != previous ) { diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy index f9342d1099..313f77dce5 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy @@ -147,20 +147,21 @@ class OperatorImpl { switch( result ) { case Collection: - result.each { it -> Op.bind(proc, target,it) } + Op.bind(proc, target, new ArrayList(result)) break case (Object[]): - result.each { it -> Op.bind(proc, target,it) } + Op.bind(proc, target, Arrays.asList(result)) break case Map: - result.each { it -> Op.bind(proc, target,it) } + Op.bind(proc, target, result.collect { it -> Op.bind(proc, target,it) }) break case Map.Entry: - Op.bind(proc, target, (result as Map.Entry).key ) - Op.bind(proc, target, (result as Map.Entry).value ) + final k = (result as Map.Entry).key + final v = (result as Map.Entry).value + Op.bind(proc, target, List.of(k,v)) break case Channel.VOID: diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy index f180d57212..382dc56007 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy @@ -26,6 +26,7 @@ import groovyx.gpars.dataflow.operator.DataflowProcessor import groovyx.gpars.dataflow.operator.PoisonPill import nextflow.Global import nextflow.Session +import nextflow.prov.OperatorRun import nextflow.prov.Prov import nextflow.prov.Tracker /** @@ -55,17 +56,23 @@ class Op { obj instanceof Tracker.Msg ? obj : Tracker.Msg.of(obj) } - static void bind(DataflowProcessor operator, DataflowWriteChannel channel, Object msg) { + static void bind(DataflowProcessor operator, DataflowWriteChannel channel, List messages) { try { - if( msg instanceof PoisonPill ) { - channel.bind(msg); - context.remove(operator); - } - else { - final ctx = context.get(operator) - if( !ctx ) - throw new IllegalStateException("Cannot find any context for operator=$operator") - Prov.getTracker().bindOutput(ctx.getPreviousRun(), channel, msg) + OperatorRun run=null + for(Object msg : messages) { + if( msg instanceof PoisonPill ) { + channel.bind(msg) + context.remove(operator) + } + else { + if( run==null ) { + final ctx = context.get(operator) + if( !ctx ) + throw new IllegalStateException("Cannot find any context for operator=$operator") + run = ctx.getOperatorRun() + } + Prov.getTracker().bindOutput(run, channel, msg) + } } } catch (Throwable t) { @@ -74,10 +81,15 @@ class Op { } } - static OpClosure instrument(Closure op, boolean accumulator=false) { + + static void bind(DataflowProcessor operator, DataflowWriteChannel channel, Object msg) { + bind(operator, channel, List.of(msg)) + } + + static OpAbstractClosure instrument(Closure op, boolean accumulator=false) { return accumulator - ? new OpAccumulatorClosure(op) - : new OpClosure(op) + ? new OpGroupingClosure(op) + : new OpRunningClosure(op) } } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/op/OpClosure.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/op/OpAbstractClosure.groovy similarity index 80% rename from modules/nextflow/src/main/groovy/nextflow/extension/op/OpClosure.groovy rename to modules/nextflow/src/main/groovy/nextflow/extension/op/OpAbstractClosure.groovy index efca7f304d..2c2b77da3a 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/op/OpClosure.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/op/OpAbstractClosure.groovy @@ -17,7 +17,6 @@ package nextflow.extension.op -import java.util.concurrent.ConcurrentHashMap import groovy.transform.CompileStatic import groovy.util.logging.Slf4j @@ -32,30 +31,21 @@ import org.codehaus.groovy.runtime.InvokerHelper */ @Slf4j @CompileStatic -class OpClosure extends Closure implements OpContext { +abstract class OpAbstractClosure extends Closure implements OpContext { private final Closure target - private final ThreadLocal runPerThread = new ThreadLocal<>() - - private final Map holder = new ConcurrentHashMap<>() - - @Override - OperatorRun getPreviousRun() { - return holder.get('previousRun') - } - - protected void setPreviousRun(OperatorRun run) { - holder.put('previousRun', run) - } - - OpClosure(Closure code) { + OpAbstractClosure(Closure code) { super(code.getOwner(), code.getThisObject()) this.target = code this.target.setDelegate(code.getDelegate()) this.target.setResolveStrategy(code.getResolveStrategy()) } + abstract OperatorRun getOperatorRun() + + abstract protected OperatorRun allocateRun() + @Override Class[] getParameterTypes() { return target.getParameterTypes() @@ -96,13 +86,6 @@ class OpClosure extends Closure implements OpContext { target.setProperty(propertyName, newValue) } - protected OperatorRun allocateRun() { - final result = new OperatorRun() - runPerThread.set(result) - setPreviousRun(result) - return result - } - @Override Object call(final Object... args) { // when the accumulator flag true, re-use the previous run object diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/op/OpContext.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/op/OpContext.groovy index 3fd775c068..e441f85f40 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/op/OpContext.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/op/OpContext.groovy @@ -25,5 +25,5 @@ import nextflow.prov.OperatorRun * @author Paolo Di Tommaso */ interface OpContext { - OperatorRun getPreviousRun() + OperatorRun getOperatorRun() } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/op/OpAccumulatorClosure.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/op/OpGroupingClosure.groovy similarity index 62% rename from modules/nextflow/src/main/groovy/nextflow/extension/op/OpAccumulatorClosure.groovy rename to modules/nextflow/src/main/groovy/nextflow/extension/op/OpGroupingClosure.groovy index d60306dc89..6ab75f0cad 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/op/OpAccumulatorClosure.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/op/OpGroupingClosure.groovy @@ -17,26 +17,38 @@ package nextflow.extension.op -import nextflow.prov.OperatorRun +import java.util.concurrent.ConcurrentHashMap +import groovy.transform.CompileStatic +import nextflow.prov.OperatorRun /** * A closure that wraps the execution of an operator target code (closure) * and maps the inputs and outputs to the corresponding operator run. * - * This class extends {@link OpClosure} assuming that all results are `"accumulated"` + * This class extends {@link OpAbstractClosure} assuming that all results are `"accumulated"` * as they were executed in the same operator run * * @author Paolo Di Tommaso */ -class OpAccumulatorClosure extends OpClosure { +@CompileStatic +class OpGroupingClosure extends OpAbstractClosure { + + private final Map holder = new ConcurrentHashMap<>(1) - OpAccumulatorClosure(Closure code) { + OpGroupingClosure(Closure code) { super(code) - setPreviousRun(new OperatorRun()) + holder.put('run', new OperatorRun()) + } + + @Override + OperatorRun getOperatorRun() { + final result = holder.get('run') + holder.put('run', new OperatorRun()) + return result } @Override protected OperatorRun allocateRun() { - return getPreviousRun() + return holder.get('run') } } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/op/OpRunningClosure.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/op/OpRunningClosure.groovy new file mode 100644 index 0000000000..9a1487be81 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/extension/op/OpRunningClosure.groovy @@ -0,0 +1,52 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.extension.op + +import java.util.concurrent.ConcurrentHashMap + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.prov.OperatorRun +/** + * + * @author Paolo Di Tommaso + */ +@Slf4j +@CompileStatic +class OpRunningClosure extends OpAbstractClosure { + + private final Map holder = new ConcurrentHashMap<>(1) + + OpRunningClosure(Closure code) { + super(code) + } + + @Override + protected OperatorRun allocateRun() { + final result = new OperatorRun() + holder.put('run', result) + return result + } + + @Override + OperatorRun getOperatorRun() { + final result = holder.get('run') + return result + } + +} diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/op/OpClosureTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/op/OpClosureTest.groovy index 277eb55d8f..ffdf0054fd 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/op/OpClosureTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/op/OpClosureTest.groovy @@ -28,14 +28,14 @@ class OpClosureTest extends Specification { def 'should invoke target closure' () { given: def code = { a,b -> a+b } - def wrapper = new OpClosure(code) + def wrapper = new OpRunningClosure(code) when: def result = wrapper.call(1,2) then: result == 3 and: - wrapper.getPreviousRun() != null + wrapper.getOperatorRun() != null } } From fee34c70c2166843b4181338bf4608171a5e6ceb Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sat, 11 Jan 2025 17:58:16 +0100 Subject: [PATCH 21/66] Bind many Signed-off-by: Paolo Di Tommaso --- .../main/groovy/nextflow/extension/OperatorImpl.groovy | 8 ++++---- .../src/main/groovy/nextflow/extension/op/Op.groovy | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy index 313f77dce5..405802cd01 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy @@ -147,21 +147,21 @@ class OperatorImpl { switch( result ) { case Collection: - Op.bind(proc, target, new ArrayList(result)) + Op.bindMany(proc, target, new ArrayList(result)) break case (Object[]): - Op.bind(proc, target, Arrays.asList(result)) + Op.bindMany(proc, target, Arrays.asList(result)) break case Map: - Op.bind(proc, target, result.collect { it -> Op.bind(proc, target,it) }) + Op.bindMany(proc, target, result.collect { it -> Op.bind(proc, target,it) }) break case Map.Entry: final k = (result as Map.Entry).key final v = (result as Map.Entry).value - Op.bind(proc, target, List.of(k,v)) + Op.bindMany(proc, target, List.of(k,v)) break case Channel.VOID: diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy index 382dc56007..10d380e644 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy @@ -56,7 +56,7 @@ class Op { obj instanceof Tracker.Msg ? obj : Tracker.Msg.of(obj) } - static void bind(DataflowProcessor operator, DataflowWriteChannel channel, List messages) { + static void bindMany(DataflowProcessor operator, DataflowWriteChannel channel, List messages) { try { OperatorRun run=null for(Object msg : messages) { @@ -83,7 +83,7 @@ class Op { static void bind(DataflowProcessor operator, DataflowWriteChannel channel, Object msg) { - bind(operator, channel, List.of(msg)) + bindMany(operator, channel, List.of(msg)) } static OpAbstractClosure instrument(Closure op, boolean accumulator=false) { From bf06bf6583813c1e137069d6aba5ca9b431419a4 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sat, 11 Jan 2025 18:08:49 +0100 Subject: [PATCH 22/66] Add support for buffer operator Signed-off-by: Paolo Di Tommaso --- .../groovy/nextflow/extension/BufferOp.groovy | 33 ++-- .../nextflow/extension/OperatorImpl.groovy | 14 +- .../nextflow/extension/BufferOpTest.groovy | 15 +- .../test/groovy/nextflow/prov/ProvTest.groovy | 187 +++++++++++------- 4 files changed, 139 insertions(+), 110 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/BufferOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/BufferOp.groovy index 896381dc25..b095941075 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/BufferOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/BufferOp.groovy @@ -16,6 +16,9 @@ package nextflow.extension +import static nextflow.extension.DataflowHelper.* +import static nextflow.util.CheckHelper.* + import groovy.transform.CompileDynamic import groovy.transform.CompileStatic import groovy.util.logging.Slf4j @@ -28,10 +31,10 @@ import groovyx.gpars.dataflow.operator.PoisonPill import nextflow.Channel import nextflow.Global import nextflow.Session +import nextflow.extension.op.Op import org.codehaus.groovy.runtime.callsite.BooleanReturningMethodInvoker -import static nextflow.extension.DataflowHelper.newOperator -import static nextflow.util.CheckHelper.checkParams /** + * Implements the "buffer" operator * * @author Paolo Di Tommaso */ @@ -162,18 +165,14 @@ class BufferOp { @Override Object controlMessageArrived(final DataflowProcessor processor, final DataflowReadChannel channel, final int index, final Object message) { if( message instanceof PoisonPill && remainder && buffer.size() ) { - target.bind(buffer) + Op.bind(processor,target, buffer) } return message } @Override - void afterRun(DataflowProcessor processor, List messages) { - if( !stopOnFirst ) - return - if( remainder && buffer) - target.bind(buffer) - target.bind(Channel.STOP) + void afterStop(DataflowProcessor processor) { + Op.bind(processor, target, Channel.STOP) } @Override @@ -188,7 +187,11 @@ class BufferOp { boolean isOpen = startingCriteria == null // -- the operator collecting the elements - newOperator( source, target, listener ) { + final params = new OpParams() + .withInput(source) + .withListener(listener) + .withAccumulator(true) + newOperator( params ) { if( isOpen ) { buffer << it } @@ -196,14 +199,18 @@ class BufferOp { isOpen = true buffer << it } - + final proc = getDelegate() as DataflowProcessor if( closeCriteria.call(it) ) { - ((DataflowProcessor) getDelegate()).bindOutput(buffer); + Op.bind(proc, target, buffer) buffer = [] // when a *startingCriteria* is defined, close the open frame flag isOpen = (startingCriteria == null) } - + if( stopOnFirst ) { + if( remainder && buffer ) + Op.bind(proc, target, buffer) + proc.terminate() + } } } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy index 405802cd01..6a36486e80 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy @@ -661,37 +661,29 @@ class OperatorImpl { * @return A newly created dataflow queue which emitted the gathered values as bundles */ DataflowWriteChannel buffer( final DataflowReadChannel source, Map params=null, Object closingCriteria ) { - - def target = new BufferOp(source) + return new BufferOp(source) .setParams(params) .setCloseCriteria(closingCriteria) .apply() - return target } DataflowWriteChannel buffer( final DataflowReadChannel source, Object startingCriteria, Object closingCriteria ) { assert startingCriteria != null assert closingCriteria != null - def target = new BufferOp(source) + return new BufferOp(source) .setStartCriteria(startingCriteria) .setCloseCriteria(closingCriteria) .apply() - - return target } DataflowWriteChannel buffer( DataflowReadChannel source, Map params ) { checkParams( 'buffer', params, 'size','skip','remainder' ) - - def target = new BufferOp(source) + return new BufferOp(source) .setParams(params) .apply() - - return target } - DataflowWriteChannel collate( DataflowReadChannel source, int size, boolean keepRemainder = true ) { if( size <= 0 ) { throw new IllegalArgumentException("Illegal argument 'size' for operator 'collate' -- it must be greater than zero: $size") diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/BufferOpTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/BufferOpTest.groovy index 00db3c1d4d..469fa08093 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/BufferOpTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/BufferOpTest.groovy @@ -27,7 +27,7 @@ import nextflow.Session * * @author Paolo Di Tommaso */ -@Timeout(10) +@Timeout(5) class BufferOpTest extends Specification { def setup() { @@ -35,7 +35,6 @@ class BufferOpTest extends Specification { } def testBufferClose() { - when: def r1 = Channel.of(1,2,3,1,2,3).buffer({ it == 2 }) then: @@ -49,11 +48,9 @@ class BufferOpTest extends Specification { r2.unwrap() == ['a','b'] r2.unwrap() == ['c','a','b'] r2.unwrap() == Channel.STOP - } def testBufferWithCount() { - when: def r1 = Channel.of(1,2,3,1,2,3,1).buffer( size:2 ) then: @@ -71,7 +68,6 @@ class BufferOpTest extends Specification { r1.unwrap() == [1] r1.unwrap() == Channel.STOP - when: def r2 = Channel.of(1,2,3,4,5,1,2,3,4,5,1,2,9).buffer( size:3, skip:2 ) then: @@ -86,22 +82,18 @@ class BufferOpTest extends Specification { r2.unwrap() == [3,4,5] r2.unwrap() == [9] r2.unwrap() == Channel.STOP - } def testBufferInvalidArg() { - when: Channel.create().buffer( xxx: true ) then: IllegalArgumentException e = thrown() - } def testBufferOpenClose() { - when: def r1 = Channel.of(1,2,3,4,5,1,2,3,4,5,1,2).buffer( 2, 4 ) then: @@ -115,11 +107,9 @@ class BufferOpTest extends Specification { r2.unwrap() == ['a','b'] r2.unwrap() == ['a','b'] r2.unwrap() == Channel.STOP - } def testBufferCloseWithOptions() { - when: def sum = 0 def r1 = Channel.of(1,2,3,1,2,3).buffer(remainder: true, { sum+=it; sum==7 }) @@ -127,11 +117,9 @@ class BufferOpTest extends Specification { r1.unwrap() == [1,2,3,1] r1.unwrap() == [2,3] r1.unwrap() == Channel.STOP - } def testBufferWithValueChannel() { - when: def result = Channel.value(1).buffer(size: 1) then: @@ -150,5 +138,4 @@ class BufferOpTest extends Specification { result.unwrap() == Channel.STOP } - } diff --git a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy index cde9819dd2..a198bcbff2 100644 --- a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy @@ -5,11 +5,14 @@ import static test.TestHelper.* import nextflow.config.ConfigParser import nextflow.processor.TaskId import nextflow.processor.TaskProcessor +import spock.lang.Ignore +import spock.lang.Timeout import test.Dsl2Spec /** * * @author Paolo Di Tommaso */ +@Timeout(5) class ProvTest extends Dsl2Spec { def setup() { @@ -46,9 +49,8 @@ class ProvTest extends Dsl2Spec { ''') then: - def upstream = upstreamTasksOf('p2') - upstream.size() == 1 - upstream.first.name == 'p1' + upstreamTasksOf('p2') + .name == ['p1'] } def 'should track provenance with branch operator'() { @@ -85,19 +87,16 @@ class ProvTest extends Dsl2Spec { } ''') then: - def t1 = upstreamTasksOf('p2 (1)') - t1.first.name == 'p1 (1)' - t1.size() == 1 + upstreamTasksOf('p2 (1)') + .name == ['p1 (1)'] and: - def t2 = upstreamTasksOf('p3 (1)') - t2.first.name == 'p1 (2)' - t2.size() == 1 + upstreamTasksOf('p3 (1)') + .name == ['p1 (2)'] and: - def t3 = upstreamTasksOf('p3 (2)') - t3.first.name == 'p1 (3)' - t3.size() == 1 + upstreamTasksOf('p3 (2)') + .name == ['p1 (3)'] } def 'should track provenance with flatMap operator' () { @@ -124,24 +123,20 @@ class ProvTest extends Dsl2Spec { } ''') then: - def t1 = upstreamTasksOf('p2 (1)') - t1.first.name == 'p1 (1)' - t1.size() == 1 - + upstreamTasksOf('p2 (1)') + .name == ['p1 (1)'] + and: - def t2 = upstreamTasksOf('p2 (2)') - t2.first.name == 'p1 (1)' - t2.size() == 1 + upstreamTasksOf('p2 (2)') + .name == ['p1 (1)'] and: - def t3 = upstreamTasksOf('p2 (3)') - t3.first.name == 'p1 (2)' - t3.size() == 1 + upstreamTasksOf('p2 (3)') + .name == ['p1 (2)'] and: - def t4 = upstreamTasksOf('p2 (4)') - t4.first.name == 'p1 (2)' - t4.size() == 1 + upstreamTasksOf('p2 (4)') + .name == ['p1 (2)'] } def 'should track the provenance of two processes and reduce operator'() { @@ -170,8 +165,8 @@ class ProvTest extends Dsl2Spec { ''') then: - def t1 = upstreamTasksOf('p2') - t1.name == ['p1 (1)', 'p1 (2)', 'p1 (3)'] + upstreamTasksOf('p2') + .name == ['p1 (1)', 'p1 (2)', 'p1 (3)'] } def 'should track the provenance of two tasks and collectFile operator' () { @@ -199,8 +194,8 @@ class ProvTest extends Dsl2Spec { ''') then: - def t1 = upstreamTasksOf('p2 (1)') - t1.name == ['p1 (1)', 'p1 (2)', 'p1 (3)'] + upstreamTasksOf('p2 (1)') + .name == ['p1 (1)', 'p1 (2)', 'p1 (3)'] } @@ -226,8 +221,8 @@ class ProvTest extends Dsl2Spec { ''') then: - def t1 = upstreamTasksOf('p2') - t1.name == ['p1 (1)', 'p1 (2)', 'p1 (3)'] + upstreamTasksOf('p2') + .name == ['p1 (1)', 'p1 (2)', 'p1 (3)'] } @@ -253,8 +248,8 @@ class ProvTest extends Dsl2Spec { ''') then: - def t1 = upstreamTasksOf('p2') - t1.name == ['p1 (1)', 'p1 (2)', 'p1 (3)'] + upstreamTasksOf('p2') + .name == ['p1 (1)', 'p1 (2)', 'p1 (3)'] } @@ -281,13 +276,11 @@ class ProvTest extends Dsl2Spec { ''') then: - def upstream1 = upstreamTasksOf('p2 (1)') - upstream1.size() == 1 - upstream1.first.name == 'p1 (1)' + upstreamTasksOf('p2 (1)') + .name == ['p1 (1)'] then: - def upstream2 = upstreamTasksOf('p2 (2)') - upstream2.size() == 1 - upstream2.first.name == 'p1 (3)' + upstreamTasksOf('p2 (2)') + .name == ['p1 (3)'] } def 'should track provenance with unique operator'() { @@ -313,19 +306,16 @@ class ProvTest extends Dsl2Spec { ''') then: - def upstream1 = upstreamTasksOf('p2 (1)') - upstream1.size() == 1 - upstream1.first.name == 'p1 (1)' + upstreamTasksOf('p2 (1)') + .name == ['p1 (1)'] then: - def upstream2 = upstreamTasksOf('p2 (2)') - upstream2.size() == 1 - upstream2.first.name == 'p1 (2)' + upstreamTasksOf('p2 (2)') + .name == ['p1 (2)'] then: - def upstream3 = upstreamTasksOf('p2 (3)') - upstream3.size() == 1 - upstream3.first.name == 'p1 (4)' + upstreamTasksOf('p2 (3)') + .name == ['p1 (4)'] } @@ -357,14 +347,12 @@ class ProvTest extends Dsl2Spec { upstream1.first.name == 'p1 (1)' then: - def upstream2 = upstreamTasksOf('p2 (2)') - upstream2.size() == 1 - upstream2.first.name == 'p1 (2)' + upstreamTasksOf('p2 (2)') + .name == ['p1 (2)'] then: - def upstream3 = upstreamTasksOf('p2 (3)') - upstream3.size() == 1 - upstream3.first.name == 'p1 (5)' + upstreamTasksOf('p2 (3)') + .name == ['p1 (5)'] } def 'should track provenance with first operator'() { @@ -389,9 +377,8 @@ class ProvTest extends Dsl2Spec { ''') then: - def upstream1 = upstreamTasksOf('p2') - upstream1.size() == 1 - upstream1.first.name == 'p1 (1)' + upstreamTasksOf('p2') + .name == ['p1 (1)'] } def 'should track provenance with take operator'() { @@ -416,13 +403,11 @@ class ProvTest extends Dsl2Spec { ''') then: - def upstream1 = upstreamTasksOf('p2 (1)') - upstream1.size() == 1 - upstream1.first.name == 'p1 (1)' + upstreamTasksOf('p2 (1)') + .name == ['p1 (1)'] then: - def upstream2 = upstreamTasksOf('p2 (2)') - upstream2.size() == 1 - upstream2.first.name == 'p1 (2)' + upstreamTasksOf('p2 (2)') + .name == ['p1 (2)'] } @@ -448,9 +433,8 @@ class ProvTest extends Dsl2Spec { ''') then: - def upstream1 = upstreamTasksOf('p2') - upstream1.size() == 1 - upstream1.first.name == 'p1 (5)' + upstreamTasksOf('p2') + .name == ['p1 (5)'] } def 'should track provenance with collect operator'() { @@ -475,8 +459,8 @@ class ProvTest extends Dsl2Spec { ''') then: - def t1 = upstreamTasksOf('p2') - t1.name == ['p1 (1)', 'p1 (2)', 'p1 (3)'] + upstreamTasksOf('p2') + .name == ['p1 (1)', 'p1 (2)', 'p1 (3)'] } def 'should track provenance with count value operator'() { @@ -501,8 +485,8 @@ class ProvTest extends Dsl2Spec { ''') then: - def t1 = upstreamTasksOf('p2') - t1.name == ['p1'] + upstreamTasksOf('p2') + .name == ['p1'] } def 'should track provenance with count many operator'() { @@ -527,8 +511,67 @@ class ProvTest extends Dsl2Spec { ''') then: - def t1 = upstreamTasksOf('p2') - t1.name == ['p1 (1)', 'p1 (2)', 'p1 (3)'] + upstreamTasksOf('p2') + .name == ['p1 (1)', 'p1 (2)', 'p1 (3)'] } + @Ignore // this should be review + def 'should track provenance with min operator'() { + when: + dsl_eval(globalConfig(), ''' + workflow { + channel.of(3,2,1) | p1 | min | p2 + } + + process p1 { + input: val(x) + output: val(y) + exec: + y = x + } + + process p2 { + input: val(x) + exec: + println x + } + ''') + + then: + upstreamTasksOf('p2') + .name == ['p1 (3)'] + } + + def 'should track provenance with buffer operator'() { + when: + dsl_eval(globalConfig(), ''' + workflow { + channel.of(1,2,3,4,5) | p1 | buffer(size:2, remainder:true) | p2 + } + + process p1 { + input: val(x) + output: val(y) + exec: + y = x + } + + process p2 { + input: val(x) + exec: + println x + } + ''') + + then: + upstreamTasksOf('p2 (1)') + .name == ['p1 (1)', 'p1 (2)'] + and: + upstreamTasksOf('p2 (2)') + .name == ['p1 (3)', 'p1 (4)'] + + and: + upstreamTasksOf('p2 (3)') + .name == ['p1 (5)'] + } } From 6d13fbc983b6dae65ff23939009b2c31ed418d91 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sat, 11 Jan 2025 18:24:45 +0100 Subject: [PATCH 23/66] Add mix operator Signed-off-by: Paolo Di Tommaso --- .../groovy/nextflow/extension/MixOp.groovy | 10 +++-- .../test/groovy/nextflow/prov/ProvTest.groovy | 38 +++++++++++++++++++ 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/MixOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/MixOp.groovy index 37e49d8e45..52f4e1a9a6 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/MixOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/MixOp.groovy @@ -24,7 +24,9 @@ import java.util.concurrent.atomic.AtomicInteger import groovy.transform.CompileStatic import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowWriteChannel +import groovyx.gpars.dataflow.operator.DataflowProcessor import nextflow.Channel +import nextflow.extension.op.Op /** * Implements Nextflow Mix operator @@ -63,10 +65,10 @@ class MixOp { DataflowWriteChannel apply() { if( target == null ) target = CH.create() - def count = new AtomicInteger( others.size()+1 ) - def handlers = [ - onNext: { target << it }, - onComplete: { if(count.decrementAndGet()==0) { target << Channel.STOP } } + final count = new AtomicInteger( others.size()+1 ) + final handlers = [ + onNext: { DataflowProcessor proc, it -> Op.bind(proc, target, it) }, + onComplete: { DataflowProcessor proc -> if(count.decrementAndGet()==0) { Op.bind(proc, target, Channel.STOP) } } ] subscribeImpl(source, handlers) diff --git a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy index a198bcbff2..c50dd77f1f 100644 --- a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy @@ -574,4 +574,42 @@ class ProvTest extends Dsl2Spec { upstreamTasksOf('p2 (3)') .name == ['p1 (5)'] } + + @Ignore + def 'should track provenance with mix operator'() { + when: + dsl_eval(globalConfig(), ''' + workflow { + def c1 = channel.of(1,2) | p1 + def c2 = channel.of(3,4) | p2 + p1.out | mix(p2.out) | p3 + } + + process p1 { + input: val(x) + output: val(y) + exec: + y = x + } + + process p2 { + input: val(x) + output: val(y) + exec: + y = x + } + + process p3 { + input: val(x) + exec: + println x + } + ''') + + then: + upstreamTasksOf('p3 (1)') + .name == ['p1 (1)'] + + + } } From 980cf324a80af608390c294299513bebdf2c0a76 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sun, 12 Jan 2025 12:35:40 +0100 Subject: [PATCH 24/66] Add join operator Signed-off-by: Paolo Di Tommaso --- .../nextflow/extension/CombineOp.groovy | 2 +- .../nextflow/extension/DataflowHelper.groovy | 32 +++-- .../groovy/nextflow/extension/JoinOp.groovy | 68 ++++++--- .../groovy/nextflow/extension/KeyPair.groovy | 6 +- .../nextflow/extension/OperatorImpl.groovy | 11 +- .../nextflow/extension/SubscribeOp.groovy | 131 ++++++++++++++++++ ...gClosure.groovy => ContextGrouping.groovy} | 27 ++-- .../extension/op/ContextJoining.groovy | 48 +++++++ ...losure.groovy => ContextSequential.groovy} | 14 +- .../groovy/nextflow/extension/op/Op.groovy | 38 ++--- ...bstractClosure.groovy => OpClosure.groovy} | 12 +- .../nextflow/extension/op/OpContext.groovy | 1 + .../nextflow/extension/op/OpDatum.groovy} | 33 +++-- .../extension/op/OpClosureTest.groovy | 19 ++- .../test/groovy/nextflow/prov/ProvTest.groovy | 44 +++++- 15 files changed, 374 insertions(+), 112 deletions(-) create mode 100644 modules/nextflow/src/main/groovy/nextflow/extension/SubscribeOp.groovy rename modules/nextflow/src/main/groovy/nextflow/extension/op/{OpGroupingClosure.groovy => ContextGrouping.groovy} (69%) create mode 100644 modules/nextflow/src/main/groovy/nextflow/extension/op/ContextJoining.groovy rename modules/nextflow/src/main/groovy/nextflow/extension/op/{OpRunningClosure.groovy => ContextSequential.groovy} (81%) rename modules/nextflow/src/main/groovy/nextflow/extension/op/{OpAbstractClosure.groovy => OpClosure.groovy} (91%) rename modules/nextflow/src/{test/groovy/nextflow/extension/OpTest.groovy => main/groovy/nextflow/extension/op/OpDatum.groovy} (58%) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/CombineOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/CombineOp.groovy index 8ef6765faf..e4f856de14 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/CombineOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/CombineOp.groovy @@ -92,7 +92,7 @@ class CombineOp { def opts = new LinkedHashMap(2) opts.onNext = { if( pivot ) { - def pair = makeKey(pivot, it) + def pair = makeKey(pivot, it, null) emit(target, index, pair.keys, pair.values) } else { diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy index 4026b4d1d8..f530c95243 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy @@ -36,7 +36,12 @@ import nextflow.Channel import nextflow.Global import nextflow.Session import nextflow.dag.NodeMarker +import nextflow.extension.op.ContextGrouping import nextflow.extension.op.Op +import nextflow.extension.op.OpClosure +import nextflow.extension.op.ContextSequential +import nextflow.extension.op.OpContext +import nextflow.prov.OperatorRun /** * This class provides helper methods to implement nextflow operators @@ -46,11 +51,12 @@ import nextflow.extension.op.Op @Slf4j class DataflowHelper { + @CompileStatic static class OpParams { List inputs List outputs List listeners - boolean accumulator + OpContext context = new ContextSequential() OpParams() { } @@ -100,7 +106,12 @@ class DataflowHelper { } OpParams withAccumulator(boolean acc) { - this.accumulator = acc + this.context = acc ? new ContextGrouping() : new ContextSequential() + return this + } + + OpParams withContext(OpContext context) { + this.context = context return this } @@ -176,8 +187,8 @@ class DataflowHelper { @Override void afterRun(final DataflowProcessor processor, final List messages) { if( source instanceof DataflowExpression ) { - if( !(target instanceof DataflowExpression) ) - processor.bindOutput( Channel.STOP ) + if( target !instanceof DataflowExpression ) + Op.bind(processor, target, Channel.STOP ) processor.terminate() } } @@ -281,10 +292,11 @@ class DataflowHelper { assert params.listeners // create the underlying dataflow operator - final closure = Op.instrument(code, params.accumulator) + final context = params.context + final closure = new OpClosure(code, context) final group = Dataflow.retrieveCurrentDFPGroup() final operator = new DataflowOperator(group, params.toMap(), closure) - Op.context.put(operator, closure) + Op.context.put(operator, context) operator.start() // track the operator as dag node NodeMarker.appendOperator(operator) @@ -377,10 +389,12 @@ class DataflowHelper { @PackageScope @CompileStatic - static KeyPair makeKey(List pivot, entry) { + static KeyPair makeKey(List pivot, entry, OperatorRun run) { + if( run==null ) + throw new IllegalStateException("Argument 'run' cannot be null") final result = new KeyPair() - if( !(entry instanceof List) ) { + if( entry !instanceof List ) { if( pivot != [0] ) throw new IllegalArgumentException("Not a valid `by` index: $pivot") result.keys = [entry] @@ -396,7 +410,7 @@ class DataflowHelper { if( i in pivot ) result.addKey(list[i]) else - result.addValue(list[i]) + result.addValue(list[i], run) } return result diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/JoinOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/JoinOp.groovy index 3d9075ba60..4f7193fbb9 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/JoinOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/JoinOp.groovy @@ -25,9 +25,15 @@ import groovy.transform.PackageScope import groovy.util.logging.Slf4j import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowWriteChannel +import groovyx.gpars.dataflow.operator.DataflowProcessor import nextflow.Channel import nextflow.NF import nextflow.exception.AbortOperationException +import nextflow.extension.op.ContextJoining +import nextflow.extension.op.Op +import nextflow.extension.op.OpContext +import nextflow.extension.op.OpDatum +import nextflow.prov.OperatorRun import nextflow.util.CheckHelper /** * Implements {@link OperatorImpl#join} operator logic @@ -58,6 +64,8 @@ class JoinOp { private Set uniqueKeys = new LinkedHashSet() + private OpContext context = new ContextJoining() + JoinOp( DataflowReadChannel source, DataflowReadChannel target, Map params = null ) { CheckHelper.checkParams('join', params, JOIN_PARAMS) this.source = source @@ -81,7 +89,6 @@ class JoinOp { } DataflowWriteChannel apply() { - // the resulting channel final result = CH.create() // the following buffer maintains the state of collected items as a map of maps. @@ -91,12 +98,19 @@ class JoinOp { final count = 2 final stopCount = new AtomicInteger(count) - - DataflowHelper.subscribeImpl( source, handler(state, count, 0, result, stopCount, remainder) ) - DataflowHelper.subscribeImpl( target, handler(state, count, 1, result, stopCount, remainder) ) + subscribe0( source, handler(state, count, 0, result, stopCount, remainder) ) + subscribe0( target, handler(state, count, 1, result, stopCount, remainder) ) return result } + private void subscribe0(DataflowReadChannel source, Map events) { + new SubscribeOp() + .withSource(source) + .withEvents(events) + .withContext(context) + .apply() + } + /** * Returns the methods {@code OnNext} and {@code onComplete} which will implement the join logic * @@ -111,22 +125,22 @@ class JoinOp { final Map result = new HashMap<>(2) - result.onNext = { + result.onNext = { DataflowProcessor proc, Object it -> synchronized (this) { if(!failed) try { - def entries = join0(buffer, size, index, it) + final entries = join0(buffer, size, index, it) if( entries ) { - target.bind( entries.size()==1 ? entries[0] : entries ) + emitEntries(target, entries) } } catch (Exception e) { failed = true - target << Channel.STOP + Op.bind(proc, target, Channel.STOP) throw e } }} - result.onComplete = { + result.onComplete = { DataflowProcessor proc -> if( stopCount.decrementAndGet()==0 && !failed ) { try { if( remainder || failOnDuplicate ) @@ -135,13 +149,27 @@ class JoinOp { checkForMismatch(buffer) } finally { - target << Channel.STOP + Op.bind(proc, target, Channel.STOP) } }} - + return result } + private void emitEntries(DataflowWriteChannel target, List entries) { + final inputs = new ArrayList(entries.size()) + final values = new ArrayList(entries.size()) + for( Object it : entries ) { + if( it instanceof OpDatum ) { + inputs.addAll(it.run.inputIds) + values.add(it.value) + } + else + values.add(it) + } + final run = new OperatorRun(inputs) + Op.bind(run, target, values.size()==1 ? values[0] : values) + } /** * Implements the join operator logic. Basically buffers the values received on each channel by their key . @@ -171,7 +199,7 @@ class JoinOp { // before a match for it is found on another channel) // get the index key for this object - final item0 = DataflowHelper.makeKey(pivot, data) + final item0 = DataflowHelper.makeKey(pivot, data, context.getOperatorRun()) // check for unique keys checkForDuplicate(item0.keys, item0.values, index, false) @@ -187,11 +215,10 @@ class JoinOp { channels[index] = [] } - def entries = channels[index] - // add the received item to the list // when it is used in the gather op add always as the first item - entries << item0.values + final entries = channels[index] + entries.add(item0.values) setSingleton(index, item0.values.size()==0) // now check if it has received an element matching for each channel @@ -221,7 +248,7 @@ class JoinOp { return result } - private final void checkRemainder(Map> buffers, int count, DataflowWriteChannel target ) { + private final void checkRemainder(Map> buffers, int count, DataflowWriteChannel target) { log.trace "Operator `join` remainder buffer: ${-> buffers}" for( Object key : buffers.keySet() ) { @@ -249,9 +276,9 @@ class JoinOp { } if( fill ) { - final value = singleton() ? result[0] : result // bind value to target channel - if( remainder ) target.bind(value) + if( remainder ) + emitEntries(target, result) } else break @@ -293,7 +320,10 @@ class JoinOp { private String csv0(value, String sep) { - value instanceof List ? value.join(sep) : value.toString() + final result = value instanceof List + ? value.collect(it->OpDatum.unwrap(it)).join(sep) + : OpDatum.unwrap(value).toString() + return result } private boolean singleton(int i=-1) { diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/KeyPair.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/KeyPair.groovy index e794760b82..0dafbd1d22 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/KeyPair.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/KeyPair.groovy @@ -19,6 +19,8 @@ package nextflow.extension import groovy.transform.CompileStatic import groovy.transform.EqualsAndHashCode import groovy.transform.ToString +import nextflow.extension.op.OpDatum +import nextflow.prov.OperatorRun /** * Implements an helper key-value helper object used in dataflow operators @@ -36,8 +38,8 @@ class KeyPair { keys.add(safeStr(el)) } - void addValue(el) { - values.add(el) + void addValue(el, OperatorRun run) { + values.add(OpDatum.of(el,run)) } static private safeStr(key) { diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy index 6a36486e80..62ddd57e36 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy @@ -147,21 +147,20 @@ class OperatorImpl { switch( result ) { case Collection: - Op.bindMany(proc, target, new ArrayList(result)) + result.each { it -> Op.bind(proc, target,it) } break case (Object[]): - Op.bindMany(proc, target, Arrays.asList(result)) + result.each { it -> Op.bind(proc, target,it) } break case Map: - Op.bindMany(proc, target, result.collect { it -> Op.bind(proc, target,it) }) + result.each { it -> Op.bind(proc, target,it) } break case Map.Entry: - final k = (result as Map.Entry).key - final v = (result as Map.Entry).value - Op.bindMany(proc, target, List.of(k,v)) + Op.bind(proc, target, (result as Map.Entry).key ) + Op.bind(proc, target, (result as Map.Entry).value ) break case Channel.VOID: diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/SubscribeOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/SubscribeOp.groovy new file mode 100644 index 0000000000..cf258ae458 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/extension/SubscribeOp.groovy @@ -0,0 +1,131 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.extension + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import groovyx.gpars.dataflow.DataflowReadChannel +import groovyx.gpars.dataflow.expression.DataflowExpression +import groovyx.gpars.dataflow.operator.DataflowEventAdapter +import groovyx.gpars.dataflow.operator.DataflowProcessor +import nextflow.Global +import nextflow.Session +import nextflow.extension.op.OpContext + +import static nextflow.extension.DataflowHelper.newOperator +/** + * Implements the "subscribe" operator + * + * @author Paolo Di Tommaso + */ +@Slf4j +@CompileStatic +class SubscribeOp { + + private DataflowReadChannel source + private OpContext context + private Closure onNext + private Closure onComplete + private Closure onError + + private static Session getSession() { Global.getSession() as Session } + + SubscribeOp withSource(DataflowReadChannel source) { + assert source!=null + this.source = source + return this + } + + SubscribeOp withOnNext(Closure event) { + this.onNext = event + return this + } + + SubscribeOp withOnComplete(Closure event) { + this.onComplete = event + return this + } + + SubscribeOp withOnError(Closure event) { + this.onError = event + return this + } + + SubscribeOp withEvents(Map events) { + this.onNext = events.onNext as Closure + this.onComplete = events.onComplete as Closure + this.onError = events.onError as Closure + return this + } + + SubscribeOp withContext(OpContext context) { + this.context = context + return this + } + + DataflowProcessor apply() { + def error = false + def stopOnFirst = source instanceof DataflowExpression + def listener = new DataflowEventAdapter() { + + @Override + void afterStop(final DataflowProcessor processor) { + if( !onComplete || error ) return + try { + onComplete.call(processor) + } + catch( Exception e ) { + SubscribeOp.log.error("@unknown", e) + session.abort(e) + } + } + + @Override + boolean onException(final DataflowProcessor processor, final Throwable e) { + error = true + if( !onError ) { + log.error("@unknown", e) + session.abort(e) + } + else { + onError.call(e) + } + return true + } + } + + final params = new DataflowHelper.OpParams() + .withInput(source) + .withListener(listener) + .withContext(context) + + newOperator (params) { + final proc = getDelegate() as DataflowProcessor + if( onNext instanceof Closure ) { + final action = (Closure) onNext + final types = action.getParameterTypes() + types.size()==2 && types[0]==DataflowProcessor.class + ? action.call(proc, it) + : action.call(it) + } + if( stopOnFirst ) { + proc.terminate() + } + } + } +} diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/op/OpGroupingClosure.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/op/ContextGrouping.groovy similarity index 69% rename from modules/nextflow/src/main/groovy/nextflow/extension/op/OpGroupingClosure.groovy rename to modules/nextflow/src/main/groovy/nextflow/extension/op/ContextGrouping.groovy index 6ab75f0cad..897e0b8070 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/op/OpGroupingClosure.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/op/ContextGrouping.groovy @@ -20,35 +20,36 @@ package nextflow.extension.op import java.util.concurrent.ConcurrentHashMap import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j import nextflow.prov.OperatorRun + /** - * A closure that wraps the execution of an operator target code (closure) - * and maps the inputs and outputs to the corresponding operator run. - * - * This class extends {@link OpAbstractClosure} assuming that all results are `"accumulated"` - * as they were executed in the same operator run * * @author Paolo Di Tommaso */ +@Slf4j @CompileStatic -class OpGroupingClosure extends OpAbstractClosure { +class ContextGrouping implements OpContext { - private final Map holder = new ConcurrentHashMap<>(1) + private final Map holder = new ConcurrentHashMap<>(1); - OpGroupingClosure(Closure code) { - super(code) + ContextGrouping(){ holder.put('run', new OperatorRun()) } + @Override + OperatorRun allocateRun() { + final result = holder.get('run') + log.debug "+ AllocateRun=$result" + return result + } + @Override OperatorRun getOperatorRun() { final result = holder.get('run') + log.debug "+ GetOperatorRun=$result" holder.put('run', new OperatorRun()) return result } - @Override - protected OperatorRun allocateRun() { - return holder.get('run') - } } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/op/ContextJoining.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/op/ContextJoining.groovy new file mode 100644 index 0000000000..9ec3599704 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/extension/op/ContextJoining.groovy @@ -0,0 +1,48 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.extension.op + + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.prov.OperatorRun +/** + * + * @author Paolo Di Tommaso + */ +@Slf4j +@CompileStatic +class ContextJoining implements OpContext { + + private final ThreadLocal runs = ThreadLocal.withInitial(()->new OperatorRun()) + + @Override + synchronized OperatorRun allocateRun() { + final run = runs.get() + log.debug "+ AllocateRun run=$run" + return run + } + + @Override + synchronized OperatorRun getOperatorRun() { + final run = runs.get() + log.debug "+ GetOperatorRun run=$run" + runs.remove() + return run + } +} diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/op/OpRunningClosure.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/op/ContextSequential.groovy similarity index 81% rename from modules/nextflow/src/main/groovy/nextflow/extension/op/OpRunningClosure.groovy rename to modules/nextflow/src/main/groovy/nextflow/extension/op/ContextSequential.groovy index 9a1487be81..746773d62d 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/op/OpRunningClosure.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/op/ContextSequential.groovy @@ -19,25 +19,18 @@ package nextflow.extension.op import java.util.concurrent.ConcurrentHashMap -import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j import nextflow.prov.OperatorRun + /** * * @author Paolo Di Tommaso */ -@Slf4j -@CompileStatic -class OpRunningClosure extends OpAbstractClosure { +class ContextSequential implements OpContext { private final Map holder = new ConcurrentHashMap<>(1) - OpRunningClosure(Closure code) { - super(code) - } - @Override - protected OperatorRun allocateRun() { + OperatorRun allocateRun() { final result = new OperatorRun() holder.put('run', result) return result @@ -48,5 +41,4 @@ class OpRunningClosure extends OpAbstractClosure { final result = holder.get('run') return result } - } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy index 10d380e644..5f7c8ae26f 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy @@ -56,23 +56,18 @@ class Op { obj instanceof Tracker.Msg ? obj : Tracker.Msg.of(obj) } - static void bindMany(DataflowProcessor operator, DataflowWriteChannel channel, List messages) { + static void bind(DataflowProcessor operator, DataflowWriteChannel channel, Object msg) { try { - OperatorRun run=null - for(Object msg : messages) { - if( msg instanceof PoisonPill ) { - channel.bind(msg) - context.remove(operator) - } - else { - if( run==null ) { - final ctx = context.get(operator) - if( !ctx ) - throw new IllegalStateException("Cannot find any context for operator=$operator") - run = ctx.getOperatorRun() - } - Prov.getTracker().bindOutput(run, channel, msg) - } + if( msg instanceof PoisonPill ) { + channel.bind(msg) + context.remove(operator) + } + else { + final ctx = context.get(operator) + if( !ctx ) + throw new IllegalStateException("Cannot find any context for operator=$operator") + final run = ctx.getOperatorRun() + Prov.getTracker().bindOutput(run, channel, msg) } } catch (Throwable t) { @@ -81,15 +76,8 @@ class Op { } } - - static void bind(DataflowProcessor operator, DataflowWriteChannel channel, Object msg) { - bindMany(operator, channel, List.of(msg)) - } - - static OpAbstractClosure instrument(Closure op, boolean accumulator=false) { - return accumulator - ? new OpGroupingClosure(op) - : new OpRunningClosure(op) + static bind(OperatorRun run, DataflowWriteChannel channel, Object msg) { + Prov.getTracker().bindOutput(run, channel, msg) } } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/op/OpAbstractClosure.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/op/OpClosure.groovy similarity index 91% rename from modules/nextflow/src/main/groovy/nextflow/extension/op/OpAbstractClosure.groovy rename to modules/nextflow/src/main/groovy/nextflow/extension/op/OpClosure.groovy index 2c2b77da3a..16c359413d 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/op/OpAbstractClosure.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/op/OpClosure.groovy @@ -31,21 +31,19 @@ import org.codehaus.groovy.runtime.InvokerHelper */ @Slf4j @CompileStatic -abstract class OpAbstractClosure extends Closure implements OpContext { +class OpClosure extends Closure { private final Closure target + private final OpContext context - OpAbstractClosure(Closure code) { + OpClosure(Closure code, OpContext context) { super(code.getOwner(), code.getThisObject()) this.target = code this.target.setDelegate(code.getDelegate()) this.target.setResolveStrategy(code.getResolveStrategy()) + this.context = context } - abstract OperatorRun getOperatorRun() - - abstract protected OperatorRun allocateRun() - @Override Class[] getParameterTypes() { return target.getParameterTypes() @@ -89,7 +87,7 @@ abstract class OpAbstractClosure extends Closure implements OpContext { @Override Object call(final Object... args) { // when the accumulator flag true, re-use the previous run object - final OperatorRun run = allocateRun() + final OperatorRun run = context.allocateRun() // map the inputs final List inputs = Prov.getTracker().receiveInputs(run, Arrays.asList(args)) final Object result = InvokerHelper.invokeMethod(target, "call", inputs.toArray()) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/op/OpContext.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/op/OpContext.groovy index e441f85f40..92445064e4 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/op/OpContext.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/op/OpContext.groovy @@ -25,5 +25,6 @@ import nextflow.prov.OperatorRun * @author Paolo Di Tommaso */ interface OpContext { + OperatorRun allocateRun() OperatorRun getOperatorRun() } diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/OpTest.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/op/OpDatum.groovy similarity index 58% rename from modules/nextflow/src/test/groovy/nextflow/extension/OpTest.groovy rename to modules/nextflow/src/main/groovy/nextflow/extension/op/OpDatum.groovy index dff1f10c29..53f33ce428 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/OpTest.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/op/OpDatum.groovy @@ -15,27 +15,30 @@ * */ -package nextflow.extension +package nextflow.extension.op + +import groovy.transform.Canonical +import nextflow.prov.OperatorRun -import nextflow.extension.op.Op -import spock.lang.Specification /** * * @author Paolo Di Tommaso */ -class OpTest extends Specification { - - def 'should instrument a closure'() { - given: - def code = { int x, int y -> x+y } - def v1 = 1 - def v2 = 2 +@Canonical +class OpDatum { + Object value + OperatorRun run - when: - def c = Op.instrument(code) - def z = c.call([v1, v2] as Object[]) - then: - z == 3 + static OpDatum of(Object value, OperatorRun run) { + new OpDatum(value,run) + } + static Object unwrap(Object obj) { + if( obj instanceof Collection ) + return obj.collect(it-> unwrap(it)) + if( obj instanceof OpDatum ) + return obj.value + else + return obj } } diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/op/OpClosureTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/op/OpClosureTest.groovy index ffdf0054fd..d672001822 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/op/OpClosureTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/op/OpClosureTest.groovy @@ -17,8 +17,8 @@ package nextflow.extension.op -import spock.lang.Specification +import spock.lang.Specification /** * * @author Paolo Di Tommaso @@ -28,14 +28,27 @@ class OpClosureTest extends Specification { def 'should invoke target closure' () { given: def code = { a,b -> a+b } - def wrapper = new OpRunningClosure(code) + def context = new ContextSequential() + def wrapper = new OpClosure(code, context) when: def result = wrapper.call(1,2) then: result == 3 and: - wrapper.getOperatorRun() != null + context.getOperatorRun() != null } + def 'should instrument a closure'() { + given: + def code = { int x, int y -> x+y } + def v1 = 1 + def v2 = 2 + + when: + def c = new OpClosure(code, new ContextSequential()) + def z = c.call([v1, v2] as Object[]) + then: + z == 3 + } } diff --git a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy index c50dd77f1f..23b1c1761d 100644 --- a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy @@ -140,7 +140,6 @@ class ProvTest extends Dsl2Spec { } def 'should track the provenance of two processes and reduce operator'() { - when: dsl_eval(globalConfig(), ''' workflow { @@ -610,6 +609,49 @@ class ProvTest extends Dsl2Spec { upstreamTasksOf('p3 (1)') .name == ['p1 (1)'] + } + + def 'should track provenance with join operator'() { + when: + dsl_eval(globalConfig(), ''' + workflow { + def c1 = channel.of(['a',10], ['b',20]) | p1 + def c2 = channel.of(['a',11], ['b',21], ['c',31]) | p2 + p1.out | join(p2.out, remainder:true) | p3 + } + + process p1 { + input: val(x) + output: val(y) + exec: + y = x + } + + process p2 { + input: val(x) + output: val(y) + exec: + y = x + } + + process p3 { + input: val(x) + exec: + println x + } + ''') + + then: + upstreamTasksOf('p3 (1)') + .name.sort() == ['p1 (1)', 'p2 (1)'] + + and: + upstreamTasksOf('p3 (2)') + .name.sort() == ['p1 (2)', 'p2 (2)'] + + and: + upstreamTasksOf('p3 (3)') + .name.sort() == ['p2 (3)'] } } From 940b3fb55ff4ee1d278d1a6118c8d3474e6fdc8b Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sun, 12 Jan 2025 13:11:23 +0100 Subject: [PATCH 25/66] Add combine op Signed-off-by: Paolo Di Tommaso --- .../nextflow/extension/CombineOp.groovy | 46 ++++++++++++----- .../groovy/nextflow/extension/JoinOp.groovy | 4 +- ...ning.groovy => ContextRunPerThread.groovy} | 2 +- .../nextflow/extension/op/OpDatum.groovy | 12 +++-- .../nextflow/extension/CombineOpTest.groovy | 2 +- .../test/groovy/nextflow/prov/ProvTest.groovy | 49 +++++++++++++++++++ 6 files changed, 96 insertions(+), 19 deletions(-) rename modules/nextflow/src/main/groovy/nextflow/extension/op/{ContextJoining.groovy => ContextRunPerThread.groovy} (96%) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/CombineOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/CombineOp.groovy index e4f856de14..812b66ca03 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/CombineOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/CombineOp.groovy @@ -24,9 +24,17 @@ import groovy.transform.PackageScope import groovy.util.logging.Slf4j import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowWriteChannel +import groovyx.gpars.dataflow.operator.DataflowProcessor import nextflow.Channel import static nextflow.extension.DataflowHelper.addToList import static nextflow.extension.DataflowHelper.makeKey + +import nextflow.extension.op.ContextRunPerThread +import nextflow.extension.op.Op +import nextflow.extension.op.OpContext +import nextflow.extension.op.OpDatum +import nextflow.prov.OperatorRun + /** * Implements the {@link OperatorImpl#spread(groovyx.gpars.dataflow.DataflowReadChannel, java.lang.Object)} operator * @@ -54,6 +62,8 @@ class CombineOp { private List pivot = NONE + private OpContext context = new ContextRunPerThread() + CombineOp(DataflowReadChannel left, Object right) { leftChannel = left @@ -92,17 +102,17 @@ class CombineOp { def opts = new LinkedHashMap(2) opts.onNext = { if( pivot ) { - def pair = makeKey(pivot, it, null) + final pair = makeKey(pivot, it, context.getOperatorRun()) emit(target, index, pair.keys, pair.values) } else { - emit(target, index, NONE, it) + emit(target, index, NONE, OpDatum.of(it, context.getOperatorRun())) } } - opts.onComplete = { + opts.onComplete = { DataflowProcessor proc -> if( stopCount.decrementAndGet()==0) { - target << Channel.STOP + Op.bind(proc, target, Channel.STOP) }} return opts @@ -110,7 +120,7 @@ class CombineOp { @PackageScope @CompileDynamic - def tuple( List p, a, b ) { + Object tuple( List p, a, b ) { List result = new LinkedList() result.addAll(p) addToList(result, a) @@ -128,7 +138,7 @@ class CombineOp { if( index == LEFT ) { log.trace "combine >> left >> by=$p; val=$v; right-values: ${rightValues[p]}" for ( Object x : rightValues[p] ) { - target.bind( tuple(p, v, x) ) + bindValues(p, v, x) } leftValues[p].add(v) return @@ -137,7 +147,7 @@ class CombineOp { if( index == RIGHT ) { log.trace "combine >> right >> by=$p; val=$v; right-values: ${leftValues[p]}" for ( Object x : leftValues[p] ) { - target.bind( tuple(p, x, v) ) + bindValues(p, x, v) } rightValues[p].add(v) return @@ -146,19 +156,25 @@ class CombineOp { throw new IllegalArgumentException("Not a valid spread operator index: $index") } - DataflowWriteChannel apply() { + private void bindValues(List p, a, b) { + final i = new ArrayList() + final t = tuple(p, OpDatum.unwrap(a,i), OpDatum.unwrap(b,i)) + final r = new OperatorRun(i) + Op.bind(r, target, t) + } + DataflowWriteChannel apply() { target = CH.create() if( rightChannel ) { final stopCount = new AtomicInteger(2) - DataflowHelper.subscribeImpl( leftChannel, handler(LEFT, target, stopCount) ) - DataflowHelper.subscribeImpl( rightChannel, handler(RIGHT, target, stopCount) ) + subscribe0( leftChannel, handler(LEFT, target, stopCount) ) + subscribe0( rightChannel, handler(RIGHT, target, stopCount) ) } else if( rightValues != null ) { final stopCount = new AtomicInteger(1) - DataflowHelper.subscribeImpl( leftChannel, handler(LEFT, target, stopCount) ) + subscribe0( leftChannel, handler(LEFT, target, stopCount) ) } else @@ -166,4 +182,12 @@ class CombineOp { return target } + + private void subscribe0(final DataflowReadChannel source, final Map events) { + new SubscribeOp() + .withSource(source) + .withEvents(events) + .withContext(context) + .apply() + } } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/JoinOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/JoinOp.groovy index 4f7193fbb9..f2ca6a17b4 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/JoinOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/JoinOp.groovy @@ -29,7 +29,7 @@ import groovyx.gpars.dataflow.operator.DataflowProcessor import nextflow.Channel import nextflow.NF import nextflow.exception.AbortOperationException -import nextflow.extension.op.ContextJoining +import nextflow.extension.op.ContextRunPerThread import nextflow.extension.op.Op import nextflow.extension.op.OpContext import nextflow.extension.op.OpDatum @@ -64,7 +64,7 @@ class JoinOp { private Set uniqueKeys = new LinkedHashSet() - private OpContext context = new ContextJoining() + private OpContext context = new ContextRunPerThread() JoinOp( DataflowReadChannel source, DataflowReadChannel target, Map params = null ) { CheckHelper.checkParams('join', params, JOIN_PARAMS) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/op/ContextJoining.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/op/ContextRunPerThread.groovy similarity index 96% rename from modules/nextflow/src/main/groovy/nextflow/extension/op/ContextJoining.groovy rename to modules/nextflow/src/main/groovy/nextflow/extension/op/ContextRunPerThread.groovy index 9ec3599704..a34da8182b 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/op/ContextJoining.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/op/ContextRunPerThread.groovy @@ -27,7 +27,7 @@ import nextflow.prov.OperatorRun */ @Slf4j @CompileStatic -class ContextJoining implements OpContext { +class ContextRunPerThread implements OpContext { private final ThreadLocal runs = ThreadLocal.withInitial(()->new OperatorRun()) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/op/OpDatum.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/op/OpDatum.groovy index 53f33ce428..677a89a08c 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/op/OpDatum.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/op/OpDatum.groovy @@ -33,11 +33,15 @@ class OpDatum { new OpDatum(value,run) } - static Object unwrap(Object obj) { - if( obj instanceof Collection ) - return obj.collect(it-> unwrap(it)) - if( obj instanceof OpDatum ) + static Object unwrap(Object obj, List inputs=null) { + if( obj instanceof Collection ) { + return obj.collect(it-> unwrap(it,inputs)) + } + if( obj instanceof OpDatum ) { + if(inputs!=null) + inputs.addAll(obj.run.inputIds) return obj.value + } else return obj } diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/CombineOpTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/CombineOpTest.groovy index 96625c6c09..2fa27357ab 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/CombineOpTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/CombineOpTest.groovy @@ -36,7 +36,7 @@ class CombineOpTest extends Specification { def op = new CombineOp(Mock(DataflowQueue), Mock(DataflowQueue)) expect: - op.tuple(pivot, left,right) == result + op.tuple(pivot, left, right) == result where: pivot | left | right | result diff --git a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy index 23b1c1761d..be59058ec0 100644 --- a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy @@ -654,4 +654,53 @@ class ProvTest extends Dsl2Spec { .name.sort() == ['p2 (3)'] } + + + def 'should track provenance with combine operator'() { + when: + dsl_eval(globalConfig(), ''' + workflow { + def c1 = channel.of(1,2) | p1 + def c2 = channel.of('a','b') | p2 + p1.out | combine(p2.out) | p3 + } + + process p1 { + input: val(x) + output: val(y) + exec: + y = x + } + + process p2 { + input: val(x) + output: val(y) + exec: + y = x + } + + process p3 { + input: val(x) + exec: + println x + } + ''') + + then: + upstreamTasksOf('p3 (1)') + .name.sort() == ['p1 (1)', 'p2 (1)'] + + and: + upstreamTasksOf('p3 (2)') + .name.sort() == ['p1 (1)', 'p2 (2)'] + + and: + upstreamTasksOf('p3 (3)') + .name.sort() == ['p1 (2)', 'p2 (1)'] + + and: + upstreamTasksOf('p3 (4)') + .name.sort() == ['p1 (2)', 'p2 (2)'] + + } } From d85702bf6d4154c362af91f4fd1907b569959182 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sun, 12 Jan 2025 13:22:24 +0100 Subject: [PATCH 26/66] Cleanup and fix tests Signed-off-by: Paolo Di Tommaso --- .../src/main/groovy/nextflow/Session.groovy | 4 +- .../nextflow/extension/DataflowHelper.groovy | 56 ++----------------- .../groovy/nextflow/extension/KeyPair.groovy | 3 +- .../extension/op/ContextGrouping.groovy | 4 +- .../extension/op/ContextRunPerThread.groovy | 4 +- .../extension/DataflowHelperTest.groovy | 8 ++- 6 files changed, 17 insertions(+), 62 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/Session.groovy b/modules/nextflow/src/main/groovy/nextflow/Session.groovy index 5e4c3dac0a..c4db0fe29c 100644 --- a/modules/nextflow/src/main/groovy/nextflow/Session.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/Session.groovy @@ -16,7 +16,6 @@ package nextflow - import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths @@ -74,7 +73,6 @@ import nextflow.util.Duration import nextflow.util.HistoryFile import nextflow.util.LoggerHelper import nextflow.util.NameGenerator -import nextflow.util.SysHelper import nextflow.util.ThreadPoolManager import nextflow.util.Threads import nextflow.util.VersionNumber @@ -800,7 +798,7 @@ class Session implements ISession { if( status ) log.debug(status) // dump threads status - log.debug(SysHelper.dumpThreads()) +// log.debug(SysHelper.dumpThreads()) // force termination notifyError(null) ansiLogObserver?.forceTermination() diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy index f530c95243..24b08368ea 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy @@ -336,62 +336,16 @@ class DataflowHelper { static final DataflowProcessor subscribeImpl(final DataflowReadChannel source, final boolean accumulator, final Map events ) { checkSubscribeHandlers(events) - - def error = false - def stopOnFirst = source instanceof DataflowExpression - def listener = new DataflowEventAdapter() { - - @Override - void afterStop(final DataflowProcessor processor) { - if( !events.onComplete || error ) return - try { - events.onComplete.call(processor) - } - catch( Exception e ) { - OperatorImpl.log.error("@unknown", e) - session.abort(e) - } - } - - @Override - boolean onException(final DataflowProcessor processor, final Throwable e) { - error = true - if( !events.onError ) { - log.error("@unknown", e) - session.abort(e) - } - else { - events.onError.call(e) - } - return true - } - } - - final params = new OpParams() - .withInput(source) - .withListener(listener) - .withAccumulator(accumulator) - - newOperator (params) { - final proc = ((DataflowProcessor) getDelegate()) - if( events.onNext instanceof Closure ) { - final action = (Closure) events.onNext - final types = action.getParameterTypes() - types.size()==2 && types[0]==DataflowProcessor.class - ? action.call(proc, it) - : action.call(it) - } - if( stopOnFirst ) { - proc.terminate() - } - } + new SubscribeOp() + .withSource(source) + .withEvents(events) + .withContext( accumulator ? new ContextSequential() : new ContextGrouping() ) + .apply() } @PackageScope @CompileStatic static KeyPair makeKey(List pivot, entry, OperatorRun run) { - if( run==null ) - throw new IllegalStateException("Argument 'run' cannot be null") final result = new KeyPair() if( entry !instanceof List ) { diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/KeyPair.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/KeyPair.groovy index 0dafbd1d22..7aacde77c9 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/KeyPair.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/KeyPair.groovy @@ -39,7 +39,8 @@ class KeyPair { } void addValue(el, OperatorRun run) { - values.add(OpDatum.of(el,run)) + final v = run ? OpDatum.of(el,run) : el + values.add(v) } static private safeStr(key) { diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/op/ContextGrouping.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/op/ContextGrouping.groovy index 897e0b8070..fa65aae4b8 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/op/ContextGrouping.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/op/ContextGrouping.groovy @@ -40,14 +40,14 @@ class ContextGrouping implements OpContext { @Override OperatorRun allocateRun() { final result = holder.get('run') - log.debug "+ AllocateRun=$result" + log.trace "+ AllocateRun=$result" return result } @Override OperatorRun getOperatorRun() { final result = holder.get('run') - log.debug "+ GetOperatorRun=$result" + log.trace "+ GetOperatorRun=$result" holder.put('run', new OperatorRun()) return result } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/op/ContextRunPerThread.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/op/ContextRunPerThread.groovy index a34da8182b..602603fc42 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/op/ContextRunPerThread.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/op/ContextRunPerThread.groovy @@ -34,14 +34,14 @@ class ContextRunPerThread implements OpContext { @Override synchronized OperatorRun allocateRun() { final run = runs.get() - log.debug "+ AllocateRun run=$run" + log.trace "+ AllocateRun run=$run" return run } @Override synchronized OperatorRun getOperatorRun() { final run = runs.get() - log.debug "+ GetOperatorRun run=$run" + log.trace "+ GetOperatorRun run=$run" runs.remove() return run } diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/DataflowHelperTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/DataflowHelperTest.groovy index ff7edfae4e..a23781e0de 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/DataflowHelperTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/DataflowHelperTest.groovy @@ -19,6 +19,8 @@ package nextflow.extension import groovyx.gpars.dataflow.DataflowQueue import groovyx.gpars.dataflow.operator.DataflowEventListener import nextflow.Session +import nextflow.extension.op.ContextGrouping +import nextflow.extension.op.ContextSequential import spock.lang.Specification import spock.lang.Unroll /** @@ -58,7 +60,7 @@ class DataflowHelperTest extends Specification { @Unroll def 'should split entry' () { when: - def pair = DataflowHelper.makeKey(pivot, entry) + def pair = DataflowHelper.makeKey(pivot, entry, null) then: pair.keys == keys pair.values == values @@ -94,7 +96,7 @@ class DataflowHelperTest extends Specification { p2.inputs == List.of(s1) p2.outputs == List.of(t1) p2.listeners == List.of(l1) - p2.accumulator + p2.context instanceof ContextGrouping and: p2.toMap().inputs == List.of(s1) p2.toMap().outputs == List.of(t1) @@ -114,7 +116,7 @@ class DataflowHelperTest extends Specification { p3.inputs == List.of(s1,s2) p3.outputs == List.of(t1,t2) p3.listeners == List.of(l1,l2) - !p3.accumulator + p3.context instanceof ContextSequential and: p3.toMap().inputs == List.of(s1,s2) p3.toMap().outputs == List.of(t1,t2) From 7183131b089423d582e0ef96e0d419dbcfb75cfe Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sun, 12 Jan 2025 14:08:46 +0100 Subject: [PATCH 27/66] Refactor core operator Signed-off-by: Paolo Di Tommaso --- .../groovy/nextflow/extension/BufferOp.groovy | 17 +- .../groovy/nextflow/extension/ChainOp.groovy | 17 +- .../nextflow/extension/CollectFileOp.groovy | 11 +- .../nextflow/extension/CollectOp.groovy | 18 ++- .../nextflow/extension/DataflowHelper.groovy | 148 ++---------------- .../nextflow/extension/DistinctOp.groovy | 14 +- .../groovy/nextflow/extension/FirstOp.groovy | 25 +-- .../groovy/nextflow/extension/ReduceOp.groovy | 16 +- .../nextflow/extension/SubscribeOp.groovy | 34 ++-- .../groovy/nextflow/extension/TakeOp.groovy | 17 +- .../groovy/nextflow/extension/UniqueOp.groovy | 14 +- .../groovy/nextflow/extension/op/Op.groovy | 109 ++++++++++++- .../extension/DataflowHelperTest.groovy | 78 +-------- .../nextflow/extension/op/OpTest.groovy | 81 ++++++++++ 14 files changed, 300 insertions(+), 299 deletions(-) create mode 100644 modules/nextflow/src/test/groovy/nextflow/extension/op/OpTest.groovy diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/BufferOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/BufferOp.groovy index b095941075..bfa5f41bd2 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/BufferOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/BufferOp.groovy @@ -31,6 +31,7 @@ import groovyx.gpars.dataflow.operator.PoisonPill import nextflow.Channel import nextflow.Global import nextflow.Session +import nextflow.extension.op.ContextGrouping import nextflow.extension.op.Op import org.codehaus.groovy.runtime.callsite.BooleanReturningMethodInvoker /** @@ -186,12 +187,8 @@ class BufferOp { // -- open frame flag boolean isOpen = startingCriteria == null - // -- the operator collecting the elements - final params = new OpParams() - .withInput(source) - .withListener(listener) - .withAccumulator(true) - newOperator( params ) { + // -- op code + final code = { if( isOpen ) { buffer << it } @@ -212,6 +209,14 @@ class BufferOp { proc.terminate() } } + + // -- the operator collecting the elements + new Op() + .withInput(source) + .withListener(listener) + .withContext(new ContextGrouping()) + .withCode(code) + .apply() } } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/ChainOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/ChainOp.groovy index 6252b7e2ae..3550c28fc8 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/ChainOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/ChainOp.groovy @@ -22,10 +22,7 @@ import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowWriteChannel import groovyx.gpars.dataflow.operator.ChainWithClosure import groovyx.gpars.dataflow.operator.DataflowEventListener - -import static nextflow.extension.DataflowHelper.newOperator -import static nextflow.extension.DataflowHelper.OpParams - +import nextflow.extension.op.Op /** * Implements the chain operator * @@ -37,7 +34,6 @@ class ChainOp { private DataflowReadChannel source private DataflowWriteChannel target private List listeners = List.of() - private boolean accumulator private Closure action static ChainOp create() { @@ -68,11 +64,6 @@ class ChainOp { return this } - ChainOp withAccumulator(boolean value) { - this.accumulator = value - return this - } - ChainOp withAction(Closure action) { this.action = action return this @@ -83,13 +74,13 @@ class ChainOp { assert target assert action - final OpParams parameters = new OpParams() + new Op() .withInput(source) .withOutput(target) - .withAccumulator(accumulator) .withListeners(listeners) + .withCode(new ChainWithClosure(action)) + .apply() - newOperator(parameters, new ChainWithClosure(action)) return target } } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/CollectFileOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/CollectFileOp.groovy index 2627d9147d..010f5f6be2 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/CollectFileOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/CollectFileOp.groovy @@ -21,12 +21,14 @@ import static nextflow.util.CheckHelper.* import java.nio.file.Path +import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowWriteChannel import groovyx.gpars.dataflow.operator.DataflowProcessor import nextflow.Channel import nextflow.Global +import nextflow.extension.op.ContextGrouping import nextflow.extension.op.Op import nextflow.file.FileCollector import nextflow.file.FileHelper @@ -264,8 +266,15 @@ class CollectFileOp { return collector } + @CompileStatic DataflowWriteChannel apply() { - DataflowHelper.subscribeImpl( channel, true, [onNext: this.&processItem, onComplete: this.&emitItems] ) + new SubscribeOp() + .withSource(channel) + .withOnNext(this.&processItem) + .withOnComplete(this.&emitItems) + .withContext( new ContextGrouping() ) + .apply() + return result } } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/CollectOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/CollectOp.groovy index af6fbeffc8..e7e3fd569a 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/CollectOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/CollectOp.groovy @@ -23,8 +23,10 @@ import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowVariable import groovyx.gpars.dataflow.operator.DataflowProcessor import nextflow.Channel +import nextflow.extension.op.ContextGrouping import nextflow.extension.op.Op import nextflow.util.ArrayBag + /** * Implements {@link OperatorImpl#collect(groovyx.gpars.dataflow.DataflowReadChannel)} operator * @@ -55,14 +57,16 @@ class CollectOp { final result = [] final target = new DataflowVariable() - Map events = [:] - events.onNext = { append(result, it) } - events.onComplete = { DataflowProcessor processor -> - final msg = result ? new ArrayBag(normalise(result)) : Channel.STOP - Op.bind(processor, target, msg) - } + new SubscribeOp() + .withSource(source) + .withContext(new ContextGrouping()) + .withOnNext { append(result, it) } + .withOnComplete { DataflowProcessor processor -> + final msg = result ? new ArrayBag(normalise(result)) : Channel.STOP + Op.bind(processor, target, msg) + } + .apply() - DataflowHelper.subscribeImpl(source, true, events) return target } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy index 24b08368ea..5a999183e9 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy @@ -21,7 +21,6 @@ import java.lang.reflect.InvocationTargetException import groovy.transform.CompileStatic import groovy.transform.PackageScope import groovy.util.logging.Slf4j -import groovyx.gpars.dataflow.Dataflow import groovyx.gpars.dataflow.DataflowChannel import groovyx.gpars.dataflow.DataflowQueue import groovyx.gpars.dataflow.DataflowReadChannel @@ -30,19 +29,12 @@ import groovyx.gpars.dataflow.DataflowWriteChannel import groovyx.gpars.dataflow.expression.DataflowExpression import groovyx.gpars.dataflow.operator.DataflowEventAdapter import groovyx.gpars.dataflow.operator.DataflowEventListener -import groovyx.gpars.dataflow.operator.DataflowOperator import groovyx.gpars.dataflow.operator.DataflowProcessor import nextflow.Channel import nextflow.Global import nextflow.Session -import nextflow.dag.NodeMarker -import nextflow.extension.op.ContextGrouping import nextflow.extension.op.Op -import nextflow.extension.op.OpClosure -import nextflow.extension.op.ContextSequential -import nextflow.extension.op.OpContext import nextflow.prov.OperatorRun - /** * This class provides helper methods to implement nextflow operators * @@ -51,79 +43,6 @@ import nextflow.prov.OperatorRun @Slf4j class DataflowHelper { - @CompileStatic - static class OpParams { - List inputs - List outputs - List listeners - OpContext context = new ContextSequential() - - OpParams() { } - - OpParams(Map params) { - if( params.inputs ) - this.inputs = params.inputs as List - if( params.outputs ) - this.outputs = params.outputs as List - if( params.listeners ) - this.listeners = params.listeners as List - } - - OpParams withInput(DataflowReadChannel channel) { - assert channel != null - this.inputs = List.of(channel) - return this - } - - OpParams withInputs(List channels) { - assert channels != null - this.inputs = channels - return this - } - - OpParams withOutput(DataflowWriteChannel channel) { - assert channel != null - this.outputs = List.of(channel) - return this - } - - OpParams withOutputs(List channels) { - assert channels != null - this.outputs = channels - return this - } - - OpParams withListener(DataflowEventListener listener) { - assert listener != null - this.listeners = List.of(listener) - return this - } - - OpParams withListeners(List listeners) { - assert listeners != null - this.listeners = listeners - return this - } - - OpParams withAccumulator(boolean acc) { - this.context = acc ? new ContextGrouping() : new ContextSequential() - return this - } - - OpParams withContext(OpContext context) { - this.context = context - return this - } - - Map toMap() { - final ret = new HashMap() - ret.inputs = inputs ?: List.of() - ret.outputs = outputs ?: List.of() - ret.listeners = listeners ?: List.of() - return ret - } - } - private static Session getSession() { Global.getSession() as Session } /** @@ -229,13 +148,10 @@ class DataflowHelper { params.listeners = [ DEF_ERROR_LISTENER ] } - return newOperator0(new OpParams(params), code) - } - - static DataflowProcessor newOperator( OpParams params, Closure code ) { - if( !params.listeners ) - params.withListener(DEF_ERROR_LISTENER) - return newOperator0(params, code) + return new Op() + .withParams(params) + .withCode(code) + .apply() } /** @@ -247,6 +163,7 @@ class DataflowHelper { * @param outputs The list of list output {@code DataflowWriteChannel}s * @param code The closure to be executed by the operator */ + @Deprecated static DataflowProcessor newOperator( List inputs, List outputs, Closure code ) { newOperator( inputs: inputs, outputs: outputs, code ) } @@ -283,63 +200,16 @@ class DataflowHelper { params.outputs = [output] params.listeners = [listener] - return newOperator0(new OpParams(params), code) - } - - static private DataflowProcessor newOperator0(OpParams params, Closure code) { - assert params - assert params.inputs - assert params.listeners - - // create the underlying dataflow operator - final context = params.context - final closure = new OpClosure(code, context) - final group = Dataflow.retrieveCurrentDFPGroup() - final operator = new DataflowOperator(group, params.toMap(), closure) - Op.context.put(operator, context) - operator.start() - // track the operator as dag node - NodeMarker.appendOperator(operator) - if( session && session.allOperators != null ) { - session.allOperators << operator - } - return operator - } - - /* - * the list of valid subscription handlers - */ - static private VALID_HANDLERS = [ 'onNext', 'onComplete', 'onError' ] - - /** - * Verify that the map contains only valid names of subscribe handlers. - * Throws an {@code IllegalArgumentException} when an invalid name is specified - * - * @param handlers The handlers map - */ - @PackageScope - static checkSubscribeHandlers( Map handlers ) { - - if( !handlers ) { - throw new IllegalArgumentException("You must specify at least one of the following events: onNext, onComplete, onError") - } - - handlers.keySet().each { - if( !VALID_HANDLERS.contains(it) ) throw new IllegalArgumentException("Not a valid handler name: $it") - } - + return new Op() + .withParams(params) + .withCode(code) + .apply() } static final DataflowProcessor subscribeImpl(final DataflowReadChannel source, final Map events ) { - subscribeImpl(source, false, events) - } - - static final DataflowProcessor subscribeImpl(final DataflowReadChannel source, final boolean accumulator, final Map events ) { - checkSubscribeHandlers(events) new SubscribeOp() .withSource(source) .withEvents(events) - .withContext( accumulator ? new ContextSequential() : new ContextGrouping() ) .apply() } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/DistinctOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/DistinctOp.groovy index 5552b4223e..fde51bc16b 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/DistinctOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/DistinctOp.groovy @@ -22,8 +22,6 @@ import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowWriteChannel import groovyx.gpars.dataflow.operator.DataflowProcessor import nextflow.extension.op.Op -import static nextflow.extension.DataflowHelper.newOperator - /** * Implements the "distinct" operator logic * @@ -61,12 +59,8 @@ class DistinctOp { if( !target ) target = CH.createBy(source) - final params = new DataflowHelper.OpParams() - .withInput(source) - .withOutput(target) - def previous = null - newOperator(params) { + final code = { final proc = getDelegate() as DataflowProcessor final key = comparator.call(it) if( key != previous ) { @@ -75,6 +69,12 @@ class DistinctOp { } } + new Op() + .withInput(source) + .withOutput(target) + .withCode(code) + .apply() + return target } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/FirstOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/FirstOp.groovy index 4869596cee..f6dadb15a5 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/FirstOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/FirstOp.groovy @@ -17,7 +17,6 @@ package nextflow.extension -import static nextflow.extension.DataflowHelper.* import groovy.transform.CompileStatic import groovyx.gpars.dataflow.DataflowReadChannel @@ -70,17 +69,15 @@ class FirstOp { final stopOnFirst = source instanceof DataflowExpression final discriminator = new BooleanReturningMethodInvoker("isCase"); - final params = new OpParams() - .withInput(source) - .withListener(new DataflowEventAdapter() { - @Override - void afterStop(DataflowProcessor proc) { - if( stopOnFirst && !target.isBound() ) - Op.bind(proc, target, Channel.STOP) - } - }) + final listener = new DataflowEventAdapter() { + @Override + void afterStop(DataflowProcessor proc) { + if( stopOnFirst && !target.isBound() ) + Op.bind(proc, target, Channel.STOP) + } + } - newOperator(params) { + final code = { final proc = getDelegate() as DataflowProcessor final accept = discriminator.invoke(criteria, it) if( accept ) @@ -89,6 +86,12 @@ class FirstOp { proc.terminate() } + new Op() + .withInput(source) + .withListener(listener) + .withCode(code) + .apply() + return target } } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/ReduceOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/ReduceOp.groovy index 11df7178bf..decf065708 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/ReduceOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/ReduceOp.groovy @@ -17,7 +17,6 @@ package nextflow.extension -import static nextflow.extension.DataflowHelper.* import groovy.transform.CompileStatic import groovy.util.logging.Slf4j @@ -29,6 +28,7 @@ import groovyx.gpars.dataflow.operator.DataflowProcessor import nextflow.Channel import nextflow.Global import nextflow.Session +import nextflow.extension.op.ContextGrouping import nextflow.extension.op.Op /** * Implements reduce operator logic @@ -113,12 +113,7 @@ class ReduceOp { } } - final parameters = new OpParams() - .withInput(source) - .withAccumulator(true) - .withListener(listener) - - newOperator(parameters) { + final code = { final value = accum == null ? it : action.call(accum, it) final proc = getDelegate() as DataflowProcessor if( value!=Channel.VOID && value!=Channel.STOP ) { @@ -128,6 +123,13 @@ class ReduceOp { proc.terminate() } + new Op() + .withInput(source) + .withContext(new ContextGrouping()) + .withListener(listener) + .withCode(code) + .apply() + return target } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/SubscribeOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/SubscribeOp.groovy index cf258ae458..09d90b118b 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/SubscribeOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/SubscribeOp.groovy @@ -25,9 +25,8 @@ import groovyx.gpars.dataflow.operator.DataflowEventAdapter import groovyx.gpars.dataflow.operator.DataflowProcessor import nextflow.Global import nextflow.Session +import nextflow.extension.op.Op import nextflow.extension.op.OpContext - -import static nextflow.extension.DataflowHelper.newOperator /** * Implements the "subscribe" operator * @@ -37,6 +36,8 @@ import static nextflow.extension.DataflowHelper.newOperator @CompileStatic class SubscribeOp { + static private List VALID_HANDLERS = [ 'onNext', 'onComplete', 'onError' ] + private DataflowReadChannel source private OpContext context private Closure onNext @@ -67,14 +68,20 @@ class SubscribeOp { } SubscribeOp withEvents(Map events) { - this.onNext = events.onNext as Closure - this.onComplete = events.onComplete as Closure - this.onError = events.onError as Closure + if( events ) { + events.keySet().each { + if( !VALID_HANDLERS.contains(it) ) throw new IllegalArgumentException("Not a valid handler name: $it") + } + this.onNext = events.onNext as Closure + this.onComplete = events.onComplete as Closure + this.onError = events.onError as Closure + } return this } SubscribeOp withContext(OpContext context) { - this.context = context + if( context!=null ) + this.context = context return this } @@ -109,12 +116,7 @@ class SubscribeOp { } } - final params = new DataflowHelper.OpParams() - .withInput(source) - .withListener(listener) - .withContext(context) - - newOperator (params) { + final code = { final proc = getDelegate() as DataflowProcessor if( onNext instanceof Closure ) { final action = (Closure) onNext @@ -127,5 +129,13 @@ class SubscribeOp { proc.terminate() } } + + new Op() + .withInput(source) + .withListener(listener) + .withContext(context) + .withCode(code) + .apply() + } } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/TakeOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/TakeOp.groovy index b7bd3a7c6e..0abe0c777c 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/TakeOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/TakeOp.groovy @@ -17,7 +17,6 @@ package nextflow.extension -import static nextflow.extension.DataflowHelper.* import groovy.transform.CompileStatic import groovy.util.logging.Slf4j @@ -29,7 +28,6 @@ import nextflow.Channel import nextflow.Global import nextflow.Session import nextflow.extension.op.Op - /** * Implement "take" operator * @@ -73,16 +71,15 @@ class TakeOp { } } - final params = new OpParams() + new Op() .withInput(source) .withOutput(target) - if( length>0 ) - params.withListener(listener) - - newOperator(params) { - final proc = getDelegate() as DataflowProcessor - Op.bind(proc, target, it) - } + .withListener(length>0 ? listener : null) + .withCode { + final proc = getDelegate() as DataflowProcessor + Op.bind(proc, target, it) + } + .apply() return target } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/UniqueOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/UniqueOp.groovy index 8413539abc..1feac84192 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/UniqueOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/UniqueOp.groovy @@ -74,12 +74,7 @@ class UniqueOp { } } - final params = new DataflowHelper.OpParams() - .withInput(source) - .withOutput(target) - .withListener(listener) - - DataflowHelper.newOperator(params) { + final code = { final proc = getDelegate() as DataflowProcessor try { final key = comparator.call(it) @@ -95,6 +90,13 @@ class UniqueOp { } } + new Op() + .withInput(source) + .withOutput(target) + .withListener(listener) + .withCode(code) + .apply() + return target } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy index 5f7c8ae26f..9e46eab65b 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy @@ -21,11 +21,16 @@ import java.util.concurrent.ConcurrentHashMap import groovy.transform.CompileStatic import groovy.util.logging.Slf4j +import groovyx.gpars.dataflow.Dataflow +import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowWriteChannel +import groovyx.gpars.dataflow.operator.DataflowEventListener +import groovyx.gpars.dataflow.operator.DataflowOperator import groovyx.gpars.dataflow.operator.DataflowProcessor import groovyx.gpars.dataflow.operator.PoisonPill import nextflow.Global import nextflow.Session +import nextflow.dag.NodeMarker import nextflow.prov.OperatorRun import nextflow.prov.Prov import nextflow.prov.Tracker @@ -38,7 +43,7 @@ import nextflow.prov.Tracker @CompileStatic class Op { - static final public ConcurrentHashMap context = new ConcurrentHashMap<>(); + static final public ConcurrentHashMap allContexts = new ConcurrentHashMap<>(); static List unwrap(List messages) { final ArrayList result = new ArrayList<>(); @@ -60,10 +65,10 @@ class Op { try { if( msg instanceof PoisonPill ) { channel.bind(msg) - context.remove(operator) + allContexts.remove(operator) } else { - final ctx = context.get(operator) + final ctx = allContexts.get(operator) if( !ctx ) throw new IllegalStateException("Cannot find any context for operator=$operator") final run = ctx.getOperatorRun() @@ -80,4 +85,102 @@ class Op { Prov.getTracker().bindOutput(run, channel, msg) } + private static Session getSession() { Global.getSession() as Session } + + private List inputs + private List outputs + private List listeners + private OpContext context = new ContextSequential() + private Closure code + + List getInputs() { inputs } + List getOutputs() { outputs } + List getListeners() { listeners } + OpContext getContext() { context } + Closure getCode() { code } + + Op withInput(DataflowReadChannel channel) { + assert channel != null + this.inputs = List.of(channel) + return this + } + + Op withInputs(List channels) { + assert channels != null + this.inputs = channels + return this + } + + Op withOutput(DataflowWriteChannel channel) { + assert channel != null + this.outputs = List.of(channel) + return this + } + + Op withOutputs(List channels) { + assert channels != null + this.outputs = channels + return this + } + + Op withListener(DataflowEventListener listener) { + if( listener ) + this.listeners = List.of(listener) + return this + } + + Op withListeners(List listeners) { + if( listeners ) + this.listeners = listeners + return this + } + + Op withParams(Map params) { + if( params.inputs ) + this.inputs = params.inputs as List + if( params.outputs ) + this.outputs = params.outputs as List + if( params.listeners ) + this.listeners = params.listeners as List + return this + } + + Op withContext(OpContext context) { + if( context!=null ) + this.context = context + return this + } + + Op withCode(Closure code) { + this.code = code + return this + } + + Map toMap() { + final ret = new HashMap() + ret.inputs = inputs ?: List.of() + ret.outputs = outputs ?: List.of() + ret.listeners = listeners ?: List.of() + return ret + } + + DataflowProcessor apply() { + assert inputs + assert code + assert context + + // create the underlying dataflow operator + final closure = new OpClosure(code, context) + final group = Dataflow.retrieveCurrentDFPGroup() + final operator = new DataflowOperator(group, toMap(), closure) + Op.allContexts.put(operator, context) + operator.start() + // track the operator as dag node + NodeMarker.appendOperator(operator) + if( session && session.allOperators != null ) { + session.allOperators << operator + } + return operator + } + } diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/DataflowHelperTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/DataflowHelperTest.groovy index a23781e0de..d63b1d499a 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/DataflowHelperTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/DataflowHelperTest.groovy @@ -16,11 +16,8 @@ package nextflow.extension -import groovyx.gpars.dataflow.DataflowQueue -import groovyx.gpars.dataflow.operator.DataflowEventListener + import nextflow.Session -import nextflow.extension.op.ContextGrouping -import nextflow.extension.op.ContextSequential import spock.lang.Specification import spock.lang.Unroll /** @@ -29,34 +26,10 @@ import spock.lang.Unroll */ class DataflowHelperTest extends Specification { - def setupSpec() { new Session() } - def 'should subscribe handlers'() { - - when: - DataflowHelper.checkSubscribeHandlers( [:] ) - then: - thrown(IllegalArgumentException) - - when: - DataflowHelper.checkSubscribeHandlers( [ onNext:{}] ) - then: - true - - when: - DataflowHelper.checkSubscribeHandlers( [ onNext:{}, xxx:{}] ) - then: - thrown(IllegalArgumentException) - - when: - DataflowHelper.checkSubscribeHandlers( [ xxx:{}] ) - then: - thrown(IllegalArgumentException) - } - @Unroll def 'should split entry' () { when: @@ -74,53 +47,4 @@ class DataflowHelperTest extends Specification { [0] | 'A' | ['A'] | [] } - def 'should validate operator params' () { - when: - def p1 = new DataflowHelper.OpParams().toMap() - then: - p1.inputs == List.of() - p1.outputs == List.of() - p1.listeners == List.of() - - when: - def s1 = new DataflowQueue() - def t1 = new DataflowQueue() - def l1 = Mock(DataflowEventListener) - and: - def p2 = new DataflowHelper.OpParams() - .withInput(s1) - .withOutput(t1) - .withListener(l1) - .withAccumulator(true) - then: - p2.inputs == List.of(s1) - p2.outputs == List.of(t1) - p2.listeners == List.of(l1) - p2.context instanceof ContextGrouping - and: - p2.toMap().inputs == List.of(s1) - p2.toMap().outputs == List.of(t1) - p2.toMap().listeners == List.of(l1) - - when: - def s2 = new DataflowQueue() - def t2 = new DataflowQueue() - def l2 = Mock(DataflowEventListener) - and: - def p3 = new DataflowHelper.OpParams() - .withInputs([s1,s2]) - .withOutputs([t1,t2]) - .withListeners([l1,l2]) - .withAccumulator(false) - then: - p3.inputs == List.of(s1,s2) - p3.outputs == List.of(t1,t2) - p3.listeners == List.of(l1,l2) - p3.context instanceof ContextSequential - and: - p3.toMap().inputs == List.of(s1,s2) - p3.toMap().outputs == List.of(t1,t2) - p3.toMap().listeners == List.of(l1,l2) - - } } diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/op/OpTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/op/OpTest.groovy new file mode 100644 index 0000000000..123c4372d1 --- /dev/null +++ b/modules/nextflow/src/test/groovy/nextflow/extension/op/OpTest.groovy @@ -0,0 +1,81 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.extension.op + +import groovyx.gpars.dataflow.DataflowQueue +import groovyx.gpars.dataflow.operator.DataflowEventListener +import spock.lang.Specification + +/** + * + * @author Paolo Di Tommaso + */ +class OpTest extends Specification { + + def 'should validate operator params' () { + when: + def p1 = new Op().toMap() + then: + p1.inputs == List.of() + p1.outputs == List.of() + p1.listeners == List.of() + + when: + def s1 = new DataflowQueue() + def t1 = new DataflowQueue() + def l1 = Mock(DataflowEventListener) + def c1 = { return 'foo' } + and: + def p2 = new Op() + .withInput(s1) + .withOutput(t1) + .withListener(l1) + .withCode(c1) + then: + p2.inputs == List.of(s1) + p2.outputs == List.of(t1) + p2.listeners == List.of(l1) + p2.context instanceof ContextSequential + p2.code == c1 + and: + p2.toMap().inputs == List.of(s1) + p2.toMap().outputs == List.of(t1) + p2.toMap().listeners == List.of(l1) + + when: + def s2 = new DataflowQueue() + def t2 = new DataflowQueue() + def l2 = Mock(DataflowEventListener) + and: + def p3 = new Op() + .withInputs([s1,s2]) + .withOutputs([t1,t2]) + .withListeners([l1,l2]) + .withContext(new ContextGrouping()) + then: + p3.inputs == List.of(s1,s2) + p3.outputs == List.of(t1,t2) + p3.listeners == List.of(l1,l2) + p3.context instanceof ContextGrouping + and: + p3.toMap().inputs == List.of(s1,s2) + p3.toMap().outputs == List.of(t1,t2) + p3.toMap().listeners == List.of(l1,l2) + } + +} From b26c0237d3982364e4bf3cc6db61225a0d3e16a7 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sun, 12 Jan 2025 15:21:42 +0100 Subject: [PATCH 28/66] Refactor opertors Signed-off-by: Paolo Di Tommaso --- .../groovy/nextflow/extension/BranchOp.groovy | 5 +- .../groovy/nextflow/extension/ConcatOp.groovy | 12 ++-- .../groovy/nextflow/extension/DumpOp.groovy | 6 +- .../nextflow/extension/GroupTupleOp.groovy | 6 +- .../groovy/nextflow/extension/LastOp.groovy | 8 ++- .../groovy/nextflow/extension/MathOp.groovy | 6 +- .../nextflow/extension/MultiMapOp.groovy | 9 +-- .../nextflow/extension/OperatorImpl.groovy | 56 ++++++++++--------- .../nextflow/extension/PublishOp.groovy | 10 ++-- .../nextflow/extension/RandomSampleOp.groovy | 10 ++-- .../groovy/nextflow/extension/SplitOp.groovy | 9 +-- .../groovy/nextflow/extension/ToListOp.groovy | 9 +-- .../nextflow/extension/TransposeOp.groovy | 6 +- .../nextflow/splitter/SplitterFactory.groovy | 12 ++-- .../extension/OperatorImplTest.groovy | 2 +- .../plugin/hello/HelloExtension.groovy | 8 ++- 16 files changed, 101 insertions(+), 73 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/BranchOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/BranchOp.groovy index 082da3c2b7..3ce5dd68d4 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/BranchOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/BranchOp.groovy @@ -75,7 +75,10 @@ class BranchOp { def events = new HashMap(2) events.put('onNext', this.&doNext) events.put('onComplete', this.&doComplete) - DataflowHelper.subscribeImpl(source, events) + new SubscribeOp() + .withSource(source) + .withEvents(events) + .apply() return this } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/ConcatOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/ConcatOp.groovy index e2dd99bea5..2e63ad709a 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/ConcatOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/ConcatOp.groovy @@ -43,7 +43,6 @@ class ConcatOp { this.target = target } - DataflowWriteChannel apply() { final result = CH.create() final allChannels = [source] @@ -55,16 +54,19 @@ class ConcatOp { private static void append( DataflowWriteChannel result, List channels, int index ) { - def current = channels[index++] - def next = index < channels.size() ? channels[index] : null + final current = channels[index++] + final next = index < channels.size() ? channels[index] : null - def events = new HashMap(2) + final events = new HashMap(2) events.onNext = { DataflowProcessor proc, it -> Op.bind(proc, result, it) } events.onComplete = { DataflowProcessor proc -> if(next) append(result, channels, index) else Op.bind(proc, result, Channel.STOP) } - DataflowHelper.subscribeImpl(current, events) + new SubscribeOp() + .withSource(current) + .withEvents(events) + .apply() } } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/DumpOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/DumpOp.groovy index 2aae01dbab..79d69ccf82 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/DumpOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/DumpOp.groovy @@ -96,7 +96,11 @@ class DumpOp { events.onComplete = { CH.close0(target) } - DataflowHelper.subscribeImpl(source, events) + new SubscribeOp() + .withSource(source) + .withEvents(events) + .apply() + return target } } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/GroupTupleOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/GroupTupleOp.groovy index ddde2b825b..7932618d36 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/GroupTupleOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/GroupTupleOp.groovy @@ -226,7 +226,11 @@ class GroupTupleOp { /* * apply the logic to the source channel */ - DataflowHelper.subscribeImpl(channel, [onNext: this.&collect, onComplete: this.&finalise]) + new SubscribeOp() + .withSource(channel) + .withOnNext(this.&collect) + .withOnComplete(this.&finalise) + .apply() /* * return the target channel diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/LastOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/LastOp.groovy index 28e3d487f6..d7fd95631b 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/LastOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/LastOp.groovy @@ -53,9 +53,11 @@ class LastOp { target = new DataflowVariable() def last = null - final next = { last = it } - final done = { DataflowProcessor proc -> Op.bind(proc, target, last) } - DataflowHelper.subscribeImpl( source, [onNext:next, onComplete: done] ) + new SubscribeOp() + .withSource(source) + .withOnNext{ last = it } + .withOnComplete{ DataflowProcessor proc -> Op.bind(proc, target, last) } + .apply() return target } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/MathOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/MathOp.groovy index d67587701e..50e11bbc26 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/MathOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/MathOp.groovy @@ -17,7 +17,6 @@ package nextflow.extension -import static nextflow.extension.DataflowHelper.* import groovy.transform.CompileDynamic import groovy.transform.CompileStatic @@ -74,7 +73,10 @@ class MathOp { assert source!=null if( target==null ) target = new DataflowVariable() - subscribeImpl(source, [onNext: aggregate.&process, onComplete: this.&completion ]) + new SubscribeOp() + .withSource(source) + .withEvents(onNext: aggregate.&process, onComplete: this.&completion) + .apply() return target } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/MultiMapOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/MultiMapOp.groovy index 57693b6901..467c734e09 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/MultiMapOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/MultiMapOp.groovy @@ -67,10 +67,11 @@ class MultiMapOp { } MultiMapOp apply() { - def events = new HashMap(2) - events.put('onNext', this.&doNext) - events.put('onComplete', this.&doComplete) - DataflowHelper.subscribeImpl(source, events) + new SubscribeOp() + .withSource(source) + .withOnNext(this.&doNext) + .withOnComplete(this.&doComplete) + .apply() return this } } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy index 62ddd57e36..c15b8b48ed 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy @@ -68,7 +68,10 @@ class OperatorImpl { * @return */ DataflowReadChannel subscribe(final DataflowReadChannel source, final Closure closure) { - subscribeImpl( source, [onNext: closure] ) + new SubscribeOp() + .withSource(source) + .withOnNext(closure) + .apply() return source } @@ -80,7 +83,10 @@ class OperatorImpl { * @return */ DataflowReadChannel subscribe(final DataflowReadChannel source, final Map events ) { - subscribeImpl(source, events) + new SubscribeOp() + .withSource(source) + .withEvents(events) + .apply() return source } @@ -911,7 +917,11 @@ class OperatorImpl { result.bind(Channel.STOP) } - subscribeImpl(source, [onNext: next, onComplete: complete]) + new SubscribeOp() + .withSource(source) + .withOnNext(next) + .withOnComplete(complete) + .apply() return result } @@ -931,17 +941,16 @@ class OperatorImpl { final newLine = opts.newLine != false final target = CH.createBy(source); - final apply = new HashMap(2) - - apply.onNext = { - final obj = closure != null ? closure.call(it) : it - session.printConsole(obj?.toString(), newLine) - target.bind(it) - } - apply. onComplete = { CH.close0(target) } - - subscribeImpl(source,apply) + new SubscribeOp() + .withSource(source) + .withOnNext{ + final obj = closure != null ? closure.call(it) : it + session.printConsole(obj?.toString(), newLine) + target.bind(it) + } + .withOnComplete{ CH.close0(target) } + .apply() return target } @@ -974,8 +983,7 @@ class OperatorImpl { if( source instanceof DataflowExpression ) throw new IllegalArgumentException("Operator `randomSample` cannot be applied to a value channel") - final result = new RandomSampleOp(source,n, seed).apply() - return result + return new RandomSampleOp(source,n, seed).apply() } DataflowWriteChannel toInteger(final DataflowReadChannel source) { @@ -995,13 +1003,11 @@ class OperatorImpl { } DataflowWriteChannel transpose( final DataflowReadChannel source, final Map params=null ) { - def result = new TransposeOp(source,params).apply() - return result + return new TransposeOp(source,params).apply() } DataflowWriteChannel splitText(DataflowReadChannel source, Map opts=null) { - final result = new SplitOp( source, 'splitText', opts ).apply() - return result + return new SplitOp( source, 'splitText', opts ).apply() } DataflowWriteChannel splitText(DataflowReadChannel source, Map opts=null, Closure action) { @@ -1014,23 +1020,19 @@ class OperatorImpl { } DataflowWriteChannel splitCsv(DataflowReadChannel source, Map opts=null) { - final result = new SplitOp( source, 'splitCsv', opts ).apply() - return result + return new SplitOp( source, 'splitCsv', opts ).apply() } DataflowWriteChannel splitFasta(DataflowReadChannel source, Map opts=null) { - final result = new SplitOp( source, 'splitFasta', opts ).apply() - return result + return new SplitOp( source, 'splitFasta', opts ).apply() } DataflowWriteChannel splitFastq(DataflowReadChannel source, Map opts=null) { - final result = new SplitOp( source, 'splitFastq', opts ).apply() - return result + return new SplitOp( source, 'splitFastq', opts ).apply() } DataflowWriteChannel splitJson(DataflowReadChannel source, Map opts=null) { - final result = new SplitOp( source, 'splitJson', opts ).apply() - return result + return new SplitOp( source, 'splitJson', opts ).apply() } DataflowWriteChannel countLines(DataflowReadChannel source, Map opts=null) { diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/PublishOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/PublishOp.groovy index 3f10b7c16c..f1b3a1967a 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/PublishOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/PublishOp.groovy @@ -18,7 +18,6 @@ package nextflow.extension import java.nio.file.Path -import groovy.json.JsonOutput import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import groovyx.gpars.dataflow.DataflowReadChannel @@ -64,10 +63,11 @@ class PublishOp { boolean getComplete() { complete } PublishOp apply() { - final events = new HashMap(2) - events.onNext = this.&onNext - events.onComplete = this.&onComplete - DataflowHelper.subscribeImpl(source, events) + new SubscribeOp() + .withSource(source) + .withOnNext(this.&onNext) + .withOnComplete(this.&onComplete) + .apply() return this } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/RandomSampleOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/RandomSampleOp.groovy index 97c58ee2da..7b4143e0f4 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/RandomSampleOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/RandomSampleOp.groovy @@ -20,10 +20,6 @@ import groovy.transform.CompileStatic import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowWriteChannel import nextflow.Channel - -import static DataflowHelper.eventsMap -import static DataflowHelper.subscribeImpl - /** * Implements Reservoir sampling of channel content * @@ -80,7 +76,11 @@ class RandomSampleOp { DataflowWriteChannel apply() { result = CH.create() - subscribeImpl(source, eventsMap(this.&sampling, this.&emit)) + new SubscribeOp() + .withSource(source) + .withOnNext(this.&sampling) + .withOnComplete(this.&emit) + .apply() return result } } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/SplitOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/SplitOp.groovy index db0e58462e..4c6b99e685 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/SplitOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/SplitOp.groovy @@ -186,10 +186,11 @@ class SplitOp { @PackageScope void applySplittingOperator( DataflowReadChannel origin, DataflowWriteChannel output, AbstractSplitter splitter ) { - final events = new HashMap(2) - events.onNext = { entry -> splitter.target(entry).apply() } - events.onComplete = { output << Channel.STOP } - DataflowHelper.subscribeImpl ( origin, events ) + new SubscribeOp() + .withSource(origin) + .withOnNext({ entry -> splitter.target(entry).apply() }) + .withOnComplete({ output << Channel.STOP }) + .apply() } @PackageScope diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/ToListOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/ToListOp.groovy index 2247145271..4ad4e44589 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/ToListOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/ToListOp.groovy @@ -52,10 +52,11 @@ class ToListOp { final target = new DataflowVariable() if( source instanceof DataflowExpression ) { final result = new ArrayList(1) - Map events = [:] - events.onNext = { result.add(it) } - events.onComplete = { DataflowProcessor processor -> Op.bind(processor, target, result) } - DataflowHelper.subscribeImpl(source, events) + new SubscribeOp() + .withSource(source) + .withOnNext({ result.add(it) }) + .withOnComplete({ DataflowProcessor processor -> Op.bind(processor, target, result) }) + .apply() return target } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/TransposeOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/TransposeOp.groovy index e9c26a61f8..9d3a71812c 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/TransposeOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/TransposeOp.groovy @@ -64,7 +64,11 @@ class TransposeOp { } DataflowWriteChannel apply() { - DataflowHelper.subscribeImpl(source, DataflowHelper.eventsMap(this.&transpose, this.&done)) + new SubscribeOp() + .withSource(source) + .withOnNext(this.&transpose) + .withOnComplete(this.&done) + .apply() return target } diff --git a/modules/nextflow/src/main/groovy/nextflow/splitter/SplitterFactory.groovy b/modules/nextflow/src/main/groovy/nextflow/splitter/SplitterFactory.groovy index e7d1d13168..f1bb8731e3 100644 --- a/modules/nextflow/src/main/groovy/nextflow/splitter/SplitterFactory.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/splitter/SplitterFactory.groovy @@ -21,7 +21,7 @@ import groovy.util.logging.Slf4j import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowVariable import groovyx.gpars.dataflow.DataflowWriteChannel -import nextflow.extension.DataflowHelper +import nextflow.extension.SubscribeOp /** * Factory class for splitter objects * @@ -95,11 +95,11 @@ class SplitterFactory { opt.each = { count++ } strategy.options(opt) - def events = new HashMap(2) - events.onNext = { entry -> strategy.target(entry).apply() } - events.onComplete = { result.bind(count) } - - DataflowHelper.subscribeImpl ( source, events ) + new SubscribeOp() + .withSource(source) + .withOnNext({ entry -> strategy.target(entry).apply() }) + .withOnComplete({ result.bind(count) }) + .apply() // return the resulting channel return result diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/OperatorImplTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/OperatorImplTest.groovy index fbfd2adeaa..85f3380e9f 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/OperatorImplTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/OperatorImplTest.groovy @@ -91,7 +91,7 @@ class OperatorImplTest extends Specification { def channel = Channel.create() int count = 0 channel.subscribe { count++; } << 1 << 2 << 3 - sleep(100) + sleep(200) then: count == 3 diff --git a/modules/nf-commons/src/testFixtures/groovy/nextflow/plugin/hello/HelloExtension.groovy b/modules/nf-commons/src/testFixtures/groovy/nextflow/plugin/hello/HelloExtension.groovy index bbfc82a514..6f35191931 100644 --- a/modules/nf-commons/src/testFixtures/groovy/nextflow/plugin/hello/HelloExtension.groovy +++ b/modules/nf-commons/src/testFixtures/groovy/nextflow/plugin/hello/HelloExtension.groovy @@ -27,11 +27,10 @@ import nextflow.Global import nextflow.NF import nextflow.Session import nextflow.extension.CH -import nextflow.extension.DataflowHelper +import nextflow.extension.SubscribeOp import nextflow.plugin.extension.Function import nextflow.plugin.extension.Operator import nextflow.plugin.extension.PluginExtensionPoint - /** * @author : jorge * @@ -106,7 +105,10 @@ class HelloExtension extends PluginExtensionPoint { final done = { target.bind(Channel.STOP) } - DataflowHelper.subscribeImpl(source, [onNext: next, onComplete: done]) + new SubscribeOp() + .withSource(source) + .withEvents([onNext: next, onComplete: done]) + .apply() return target } From 8639e4d6986c5a410a73702020a9506a04514b85 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sun, 12 Jan 2025 15:28:58 +0100 Subject: [PATCH 29/66] Add concat operator Signed-off-by: Paolo Di Tommaso --- .../groovy/nextflow/extension/ConcatOp.groovy | 24 +++++----- .../test/groovy/nextflow/prov/ProvTest.groovy | 44 +++++++++++++++++++ 2 files changed, 55 insertions(+), 13 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/ConcatOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/ConcatOp.groovy index 2e63ad709a..83ad879c8f 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/ConcatOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/ConcatOp.groovy @@ -21,7 +21,9 @@ import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowWriteChannel import groovyx.gpars.dataflow.operator.DataflowProcessor import nextflow.Channel +import nextflow.extension.op.ContextRunPerThread import nextflow.extension.op.Op +import nextflow.extension.op.OpContext /** * Implements the {@link OperatorImpl#concat} operator @@ -35,10 +37,11 @@ class ConcatOp { private DataflowReadChannel[] target + private OpContext context = new ContextRunPerThread() + ConcatOp( DataflowReadChannel source, DataflowReadChannel... target ) { assert source != null assert target - this.source = source this.target = target } @@ -47,26 +50,21 @@ class ConcatOp { final result = CH.create() final allChannels = [source] allChannels.addAll(target) - append(result, allChannels, 0) return result } - - private static void append( DataflowWriteChannel result, List channels, int index ) { + private void append( DataflowWriteChannel result, List channels, int index ) { final current = channels[index++] final next = index < channels.size() ? channels[index] : null - - final events = new HashMap(2) - events.onNext = { DataflowProcessor proc, it -> Op.bind(proc, result, it) } - events.onComplete = { DataflowProcessor proc -> - if(next) append(result, channels, index) - else Op.bind(proc, result, Channel.STOP) - } - new SubscribeOp() .withSource(current) - .withEvents(events) + .withContext(context) + .withOnNext { DataflowProcessor proc, it -> Op.bind(proc, result, it) } + .withOnComplete { DataflowProcessor proc -> + if(next) append(result, channels, index) + else Op.bind(proc, result, Channel.STOP) + } .apply() } } diff --git a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy index be59058ec0..176e4ec0e3 100644 --- a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy @@ -703,4 +703,48 @@ class ProvTest extends Dsl2Spec { .name.sort() == ['p1 (2)', 'p2 (2)'] } + + def 'should track provenance with concat operator'() { + when: + dsl_eval(globalConfig(), ''' + workflow { + def c1 = channel.of(1,2) | p1 + def c2 = channel.of(3,4) | p2 + p1.out | concat(p2.out) | p3 + } + + process p1 { + input: val(x) + output: val(y) + exec: + y = x + } + + process p2 { + input: val(x) + output: val(y) + exec: + y = x + } + + process p3 { + input: val(x) + exec: + println x + } + ''') + + then: + upstreamTasksOf('p3 (1)') + .name == ['p1 (1)'] + and: + upstreamTasksOf('p3 (2)') + .name == ['p1 (2)'] + and: + upstreamTasksOf('p3 (3)') + .name == ['p2 (1)'] + and: + upstreamTasksOf('p3 (4)') + .name == ['p2 (2)'] + } } From c2a3e95c4c00360546851645e779feef91fff226 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sun, 12 Jan 2025 16:57:12 +0100 Subject: [PATCH 30/66] Add mix operator Signed-off-by: Paolo Di Tommaso --- .../main/groovy/nextflow/extension/MixOp.groovy | 17 +++++++++++++---- .../test/groovy/nextflow/prov/ProvTest.groovy | 4 +++- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/MixOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/MixOp.groovy index 52f4e1a9a6..c9a0cf3f89 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/MixOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/MixOp.groovy @@ -17,8 +17,6 @@ package nextflow.extension -import static nextflow.extension.DataflowHelper.* - import java.util.concurrent.atomic.AtomicInteger import groovy.transform.CompileStatic @@ -26,7 +24,9 @@ import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowWriteChannel import groovyx.gpars.dataflow.operator.DataflowProcessor import nextflow.Channel +import nextflow.extension.op.ContextRunPerThread import nextflow.extension.op.Op +import nextflow.extension.op.OpContext /** * Implements Nextflow Mix operator @@ -39,6 +39,7 @@ class MixOp { private DataflowReadChannel source private List others private DataflowWriteChannel target + private OpContext context = new ContextRunPerThread() MixOp(DataflowReadChannel source, DataflowReadChannel other) { this.source = source @@ -71,9 +72,9 @@ class MixOp { onComplete: { DataflowProcessor proc -> if(count.decrementAndGet()==0) { Op.bind(proc, target, Channel.STOP) } } ] - subscribeImpl(source, handlers) + subscribe0(source, handlers) for( def it : others ) { - subscribeImpl(it, handlers) + subscribe0(it, handlers) } final allSources = [source] @@ -81,4 +82,12 @@ class MixOp { return target } + private void subscribe0(final DataflowReadChannel source, final Map events ) { + new SubscribeOp() + .withSource(source) + .withContext(context) + .withEvents(events) + .apply() + } + } diff --git a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy index 176e4ec0e3..25438986ed 100644 --- a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy @@ -574,7 +574,6 @@ class ProvTest extends Dsl2Spec { .name == ['p1 (5)'] } - @Ignore def 'should track provenance with mix operator'() { when: dsl_eval(globalConfig(), ''' @@ -608,6 +607,9 @@ class ProvTest extends Dsl2Spec { then: upstreamTasksOf('p3 (1)') .name == ['p1 (1)'] + and: + upstreamTasksOf('p3 (2)') + .name == ['p2 (1)'] } From 4d0379b663ae8bbea585765eb78f355b907afff1 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Mon, 13 Jan 2025 15:30:24 +0100 Subject: [PATCH 31/66] Remove chain operator + refactor Signed-off-by: Paolo Di Tommaso --- .../groovy/nextflow/extension/MapOp.groovy | 12 +++++ .../nextflow/extension/OperatorImpl.groovy | 50 ++++++------------- 2 files changed, 28 insertions(+), 34 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/MapOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/MapOp.groovy index 735f8aba58..0a70ac8754 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/MapOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/MapOp.groovy @@ -38,11 +38,23 @@ class MapOp { private DataflowWriteChannel target + MapOp() {} + MapOp( final DataflowReadChannel source, final Closure mapper ) { this.source = source this.mapper = mapper } + MapOp withSource(DataflowReadChannel source) { + this.source = source + return this + } + + MapOp withMapper(Closure code) { + this.mapper = code + return this + } + MapOp setTarget( DataflowWriteChannel target ) { this.target = target return this diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy index c15b8b48ed..929324826e 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy @@ -90,36 +90,6 @@ class OperatorImpl { return source } - /** - * Chain operator, this is a synonym of {@code DataflowReadChannel.chainWith} - * - * @param source - * @param closure - * @return - */ - @Deprecated - DataflowWriteChannel chain(final DataflowReadChannel source, final Closure closure) { - final target = CH.createBy(source) - newOperator(source, target, stopErrorListener(source,target), new ChainWithClosure(closure)) - return target - } - - /** - * Chain operator, this is a synonym of {@code DataflowReadChannel.chainWith} - * - * @param source - * @param closure - * @return - */ - @Deprecated - DataflowWriteChannel chain(final DataflowReadChannel source, final Map params, final Closure closure) { - return ChainOp.create() - .withSource(source) - .withTarget(CH.createBy(source)) - .withAction(closure) - .apply() - } - /** * Transform the items emitted by a channel by applying a function to each of them * @@ -987,19 +957,31 @@ class OperatorImpl { } DataflowWriteChannel toInteger(final DataflowReadChannel source) { - return chain(source, { it -> it as Integer }) + return new MapOp() + .withSource(source) + .withMapper { it -> it as Integer } + .apply() } DataflowWriteChannel toLong(final DataflowReadChannel source) { - return chain(source, { it -> it as Long }) + return new MapOp() + .withSource(source) + .withMapper { it -> it as Long } + .apply() } DataflowWriteChannel toFloat(final DataflowReadChannel source) { - return chain(source, { it -> it as Float }) + return new MapOp() + .withSource(source) + .withMapper { it -> it as Float } + .apply() } DataflowWriteChannel toDouble(final DataflowReadChannel source) { - return chain(source, { it -> it as Double }) + return new MapOp() + .withSource(source) + .withMapper { it -> it as Double } + .apply() } DataflowWriteChannel transpose( final DataflowReadChannel source, final Map params=null ) { From a056f8a49080925eb5d221424466448ec98484d4 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 14 Jan 2025 11:53:49 +0100 Subject: [PATCH 32/66] Dump threads only with trace log level [ci fast] Signed-off-by: Paolo Di Tommaso --- modules/nextflow/src/main/groovy/nextflow/Session.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/Session.groovy b/modules/nextflow/src/main/groovy/nextflow/Session.groovy index c4db0fe29c..7c34de8cf1 100644 --- a/modules/nextflow/src/main/groovy/nextflow/Session.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/Session.groovy @@ -73,6 +73,7 @@ import nextflow.util.Duration import nextflow.util.HistoryFile import nextflow.util.LoggerHelper import nextflow.util.NameGenerator +import nextflow.util.SysHelper import nextflow.util.ThreadPoolManager import nextflow.util.Threads import nextflow.util.VersionNumber @@ -798,7 +799,8 @@ class Session implements ISession { if( status ) log.debug(status) // dump threads status -// log.debug(SysHelper.dumpThreads()) + if( log.isTraceEnabled() ) + log.trace(SysHelper.dumpThreads()) // force termination notifyError(null) ansiLogObserver?.forceTermination() From e1d3346f79540c4b162d2ec35a1516d9dcbe66e7 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 14 Jan 2025 14:47:53 +0100 Subject: [PATCH 33/66] Fix tests Signed-off-by: Paolo Di Tommaso --- .../nextflow/processor/TaskProcessor.groovy | 4 +++- .../nextflow/script/params/EachInParam.groovy | 8 +++++++- .../test/groovy/nextflow/prov/ProvTest.groovy | 18 ++---------------- 3 files changed, 12 insertions(+), 18 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy index 867ebdb3d8..72960222dc 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy @@ -561,7 +561,9 @@ class TaskProcessor { // the channel forwarding the data from the *iterator* process to the target task final linkingChannels = new ArrayList(size) - size.times { linkingChannels[it] = new DataflowQueue() } + for( int i=0; i Date: Tue, 14 Jan 2025 15:09:31 +0100 Subject: [PATCH 34/66] Fix tests Signed-off-by: Paolo Di Tommaso --- .../PluginExtensionMethodsTest.groovy | 71 ++++++++++--------- 1 file changed, 36 insertions(+), 35 deletions(-) diff --git a/modules/nf-commons/src/test/nextflow/plugin/extension/PluginExtensionMethodsTest.groovy b/modules/nf-commons/src/test/nextflow/plugin/extension/PluginExtensionMethodsTest.groovy index 82cf64ebd6..a7434efa5b 100644 --- a/modules/nf-commons/src/test/nextflow/plugin/extension/PluginExtensionMethodsTest.groovy +++ b/modules/nf-commons/src/test/nextflow/plugin/extension/PluginExtensionMethodsTest.groovy @@ -19,6 +19,7 @@ package nextflow.plugin.extension import java.nio.file.Path +import groovyx.gpars.dataflow.DataflowReadChannel import nextflow.Channel import nextflow.exception.DuplicateModuleFunctionException import nextflow.plugin.Plugins @@ -67,11 +68,11 @@ class PluginExtensionMethodsTest extends Dsl2Spec { ''' when: - def result = dsl_eval(SCRIPT_TEXT) + def result = dsl_eval(SCRIPT_TEXT) as DataflowReadChannel then: - result.val == 'Bye bye folks' - result.val == Channel.STOP + result.unwrap() == 'Bye bye folks' + result.unwrap() == Channel.STOP } def 'should execute custom operator extension/2' () { @@ -85,11 +86,11 @@ class PluginExtensionMethodsTest extends Dsl2Spec { .goodbye() ''' when: - def result = dsl_eval(SCRIPT_TEXT) + def result = dsl_eval(SCRIPT_TEXT) as DataflowReadChannel then: - result.val == 'Bye bye folks' - result.val == Channel.STOP + result.unwrap() == 'Bye bye folks' + result.unwrap() == Channel.STOP } @@ -102,12 +103,12 @@ class PluginExtensionMethodsTest extends Dsl2Spec { ''' when: - def result = dsl_eval(SCRIPT_TEXT) + def result = dsl_eval(SCRIPT_TEXT) as DataflowReadChannel then: result - result.val == 'a string'.reverse() - result.val == Channel.STOP + result.unwrap() == 'a string'.reverse() + result.unwrap() == Channel.STOP } @@ -122,12 +123,12 @@ class PluginExtensionMethodsTest extends Dsl2Spec { ''' when: - def result = dsl_eval(SCRIPT_TEXT) + def result = dsl_eval(SCRIPT_TEXT) as DataflowReadChannel then: result - result.val == 'a string'.reverse() - result.val == Channel.STOP + result.unwrap() == 'a string'.reverse() + result.unwrap() == Channel.STOP } @@ -142,13 +143,13 @@ class PluginExtensionMethodsTest extends Dsl2Spec { ''' when: - def result = dsl_eval(SCRIPT_TEXT) + def result = dsl_eval(SCRIPT_TEXT) as DataflowReadChannel then: - result.val == 100 - result.val == 200 - result.val == 300 - result.val == Channel.STOP + result.unwrap() == 100 + result.unwrap() == 200 + result.unwrap() == 300 + result.unwrap() == Channel.STOP } def 'should execute custom factory as alias extension' () { @@ -161,12 +162,12 @@ class PluginExtensionMethodsTest extends Dsl2Spec { ''' when: - def result = dsl_eval(SCRIPT_TEXT) + def result = dsl_eval(SCRIPT_TEXT) as DataflowReadChannel then: result - result.val == 'reverse this string'.reverse() - result.val == Channel.STOP + result.unwrap() == 'reverse this string'.reverse() + result.unwrap() == Channel.STOP } @@ -207,11 +208,11 @@ class PluginExtensionMethodsTest extends Dsl2Spec { def 'should execute custom functions'() { when: - def result = dsl_eval(SCRIPT_TEXT) + def result = dsl_eval(SCRIPT_TEXT) as DataflowReadChannel then: - result.val == EXPECTED - result.val == Channel.STOP + result.unwrap() == EXPECTED + result.unwrap() == Channel.STOP where: SCRIPT_TEXT | EXPECTED @@ -223,13 +224,13 @@ class PluginExtensionMethodsTest extends Dsl2Spec { def 'should call init plugin in custom functions'() { when: - def result = dsl_eval(""" + dsl_eval(""" include { sayHello } from 'plugin/nf-test-plugin-hello' sayHello() """) then: - true + noExceptionThrown() } def 'should throw function not found'() { @@ -279,11 +280,11 @@ class PluginExtensionMethodsTest extends Dsl2Spec { ''' when: - def result = dsl_eval(SCRIPT_TEXT) + def result = dsl_eval(SCRIPT_TEXT) as DataflowReadChannel then: - result.val == 'hi' - result.val == Channel.STOP + result.unwrap() == 'hi' + result.unwrap() == Channel.STOP } def 'should not include a non annotated function'() { @@ -334,10 +335,10 @@ class PluginExtensionMethodsTest extends Dsl2Spec { ''' when: - def result = dsl_eval(SCRIPT) + def result = dsl_eval(SCRIPT) as DataflowReadChannel then: - result.val == 'hi' + result.unwrap() == 'hi' } @@ -390,10 +391,10 @@ class PluginExtensionMethodsTest extends Dsl2Spec { ''' when: - def result = dsl_eval(SCRIPT) + def result = dsl_eval(SCRIPT) as DataflowReadChannel then: - result.val == 'hola' + result.unwrap() == 'hola' } def 'should execute custom functions and channel extension at the same time'() { @@ -403,11 +404,11 @@ class PluginExtensionMethodsTest extends Dsl2Spec { SCRIPT.text = SCRIPT_TEXT when: - def result = dsl_eval(SCRIPT) + def result = dsl_eval(SCRIPT) as DataflowReadChannel then: - result.val == EXPECTED - result.val == Channel.STOP + result.unwrap() == EXPECTED + result.unwrap() == Channel.STOP where: SCRIPT_TEXT | EXPECTED From dbd42ae6ebf8a1ac0984664ceb08c817930726aa Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 14 Jan 2025 16:09:41 +0100 Subject: [PATCH 35/66] Fix tests Signed-off-by: Paolo Di Tommaso --- .../lifesciences/GoogleLifeSciencesScriptLauncherTest.groovy | 2 ++ .../nextflow/cloud/google/lifesciences/bash-wrapper-gcp.txt | 1 + 2 files changed, 3 insertions(+) diff --git a/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesScriptLauncherTest.groovy b/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesScriptLauncherTest.groovy index 4273b9f30c..44bfecdfef 100644 --- a/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesScriptLauncherTest.groovy +++ b/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesScriptLauncherTest.groovy @@ -21,6 +21,7 @@ import java.nio.file.Files import nextflow.Session import nextflow.cloud.google.GoogleSpecification import nextflow.processor.TaskBean +import nextflow.processor.TaskId import nextflow.util.MustacheTemplateEngine /** * @@ -72,6 +73,7 @@ class GoogleLifeSciencesScriptLauncherTest extends GoogleSpecification { } } def bean = [name: 'Hello 1', + taskId: TaskId.of(10), script: 'echo Hello world!', workDir: WORK_DIR] as TaskBean /* diff --git a/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/bash-wrapper-gcp.txt b/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/bash-wrapper-gcp.txt index c7382062a1..5226420f71 100644 --- a/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/bash-wrapper-gcp.txt +++ b/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/bash-wrapper-gcp.txt @@ -1,5 +1,6 @@ #!/bin/bash ### --- +### id: '10' ### name: 'Hello 1' ### ... set -e From 4e154bb7b55ae4bff4cd1839e9fd1f661c6cfed6 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 14 Jan 2025 23:16:56 +0100 Subject: [PATCH 36/66] Refactor flatMap op [ci fast] Signed-off-by: Paolo Di Tommaso --- .../nextflow/extension/FlatMapOp.groovy | 99 +++++++++++++++++++ .../groovy/nextflow/extension/MapOp.groovy | 4 +- .../nextflow/extension/OperatorImpl.groovy | 48 ++------- .../groovy/nextflow/extension/op/Op.groovy | 2 +- 4 files changed, 111 insertions(+), 42 deletions(-) create mode 100644 modules/nextflow/src/main/groovy/nextflow/extension/FlatMapOp.groovy diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/FlatMapOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/FlatMapOp.groovy new file mode 100644 index 0000000000..79d4f1fe75 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/extension/FlatMapOp.groovy @@ -0,0 +1,99 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.extension + +import static nextflow.extension.DataflowHelper.* + +import groovy.transform.CompileStatic +import groovyx.gpars.dataflow.DataflowReadChannel +import groovyx.gpars.dataflow.DataflowWriteChannel +import groovyx.gpars.dataflow.operator.DataflowProcessor +import nextflow.Channel +import nextflow.extension.op.Op +/** + * Implement "flatMap" operator + * + * @author Paolo Di Tommaso + */ +@CompileStatic +class FlatMapOp { + + private DataflowReadChannel source + private DataflowWriteChannel target + private Closure mapper + + FlatMapOp withSource(DataflowReadChannel source) { + assert source!=null + this.source = source + return this + } + + FlatMapOp withMapper(Closure code) { + this.mapper = code + return this + } + + FlatMapOp setTarget( DataflowWriteChannel target ) { + this.target = target + return this + } + + DataflowWriteChannel apply() { + assert source!=null + + if( target==null ) + target = CH.create() + + new Op() + .withInput(source) + .withOutput(target) + .withListener(stopErrorListener(source,target)) + .withCode { Object item -> + final result = mapper != null ? mapper.call(item) : item + final proc = getDelegate() as DataflowProcessor + + switch( result ) { + case Collection: + result.each { it -> Op.bind(proc, target,it) } + break + + case (Object[]): + result.each { it -> Op.bind(proc, target,it) } + break + + case Map: + result.each { it -> Op.bind(proc, target,it) } + break + + case Map.Entry: + Op.bind(proc, target, (result as Map.Entry).key ) + Op.bind(proc, target, (result as Map.Entry).value ) + break + + case Channel.VOID: + break + + default: + Op.bind(proc, target, result) + } + } + .apply() + return target + } + +} diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/MapOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/MapOp.groovy index 0a70ac8754..87b0e95306 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/MapOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/MapOp.groovy @@ -45,12 +45,14 @@ class MapOp { this.mapper = mapper } - MapOp withSource(DataflowReadChannel source) { + MapOp withSource(DataflowReadChannel source) { + assert source!=null this.source = source return this } MapOp withMapper(Closure code) { + assert code!=null this.mapper = code return this } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy index 929324826e..a604c96d45 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy @@ -35,7 +35,6 @@ import nextflow.Channel import nextflow.Global import nextflow.NF import nextflow.Session -import nextflow.extension.op.Op import nextflow.script.ChannelOut import nextflow.script.TokenBranchDef import nextflow.script.TokenMultiMapDef @@ -98,9 +97,10 @@ class OperatorImpl { * @return */ DataflowWriteChannel map(final DataflowReadChannel source, final Closure closure) { - assert source != null - assert closure - return new MapOp(source, closure).apply() + return new MapOp() + .withSource(source) + .withMapper(closure) + .apply() } /** @@ -112,42 +112,10 @@ class OperatorImpl { */ DataflowWriteChannel flatMap(final DataflowReadChannel source, final Closure closure=null) { assert source != null - - final target = CH.create() - final listener = stopErrorListener(source,target) - - newOperator(source, target, listener) { Object item -> - - final result = closure != null ? closure.call(item) : item - final proc = ((DataflowProcessor) getDelegate()) - - switch( result ) { - case Collection: - result.each { it -> Op.bind(proc, target,it) } - break - - case (Object[]): - result.each { it -> Op.bind(proc, target,it) } - break - - case Map: - result.each { it -> Op.bind(proc, target,it) } - break - - case Map.Entry: - Op.bind(proc, target, (result as Map.Entry).key ) - Op.bind(proc, target, (result as Map.Entry).value ) - break - - case Channel.VOID: - break - - default: - Op.bind(proc, target, result) - } - } - - return target + new FlatMapOp() + .withSource(source) + .withMapper(closure) + .apply() } /** diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy index 9e46eab65b..f7ada40b30 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy @@ -173,7 +173,7 @@ class Op { final closure = new OpClosure(code, context) final group = Dataflow.retrieveCurrentDFPGroup() final operator = new DataflowOperator(group, toMap(), closure) - Op.allContexts.put(operator, context) + allContexts.put(operator, context) operator.start() // track the operator as dag node NodeMarker.appendOperator(operator) From 2f643f9668441010d91c28c5fd09c578466f089c Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 14 Jan 2025 23:25:18 +0100 Subject: [PATCH 37/66] Refactor [ci fast] Signed-off-by: Paolo Di Tommaso --- .../main/groovy/nextflow/extension/BranchOp.groovy | 10 +++++----- .../groovy/nextflow/extension/CollectFileOp.groovy | 8 ++++---- .../groovy/nextflow/extension/CollectOp.groovy | 4 ++-- .../groovy/nextflow/extension/CombineOp.groovy | 6 +++--- .../main/groovy/nextflow/extension/ConcatOp.groovy | 6 +++--- .../nextflow/extension/DataflowHelper.groovy | 10 +++++----- .../main/groovy/nextflow/extension/FirstOp.groovy | 4 ++-- .../main/groovy/nextflow/extension/IntoOp.groovy | 6 +++--- .../main/groovy/nextflow/extension/JoinOp.groovy | 8 ++++---- .../main/groovy/nextflow/extension/LastOp.groovy | 2 +- .../main/groovy/nextflow/extension/MathOp.groovy | 4 ++-- .../main/groovy/nextflow/extension/MixOp.groovy | 4 ++-- .../groovy/nextflow/extension/OperatorImpl.groovy | 14 +++++++------- .../main/groovy/nextflow/extension/ReduceOp.groovy | 6 +++--- .../groovy/nextflow/extension/SubscribeOp.groovy | 6 +++--- .../main/groovy/nextflow/extension/TakeOp.groovy | 8 ++++---- .../main/groovy/nextflow/extension/ToListOp.groovy | 2 +- .../main/groovy/nextflow/extension/UniqueOp.groovy | 2 +- 18 files changed, 55 insertions(+), 55 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/BranchOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/BranchOp.groovy index 3ce5dd68d4..0614fe1fd6 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/BranchOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/BranchOp.groovy @@ -52,21 +52,21 @@ class BranchOp { ChannelOut getOutput() { this.output } - protected void doNext(DataflowProcessor proc, Object it) { + protected void doNext(DataflowProcessor dp, Object it) { TokenBranchChoice ret = switchDef.closure.call(it) if( ret ) { - Op.bind(proc, targets[ret.choice], ret.value) + Op.bind(dp, targets[ret.choice], ret.value) } } - protected void doComplete(DataflowProcessor proc) { + protected void doComplete(DataflowProcessor dp) { for( DataflowWriteChannel ch : targets.values() ) { if( ch instanceof DataflowExpression ) { if( !ch.isBound() ) - Op.bind(proc, ch, Channel.STOP) + Op.bind(dp, ch, Channel.STOP) } else { - Op.bind(proc, ch, Channel.STOP) + Op.bind(dp, ch, Channel.STOP) } } } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/CollectFileOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/CollectFileOp.groovy index 010f5f6be2..efc1ac5449 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/CollectFileOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/CollectFileOp.groovy @@ -141,7 +141,7 @@ class CollectFileOp { * each time a value is received, invoke the closure and * append its result value to a file */ - protected processItem( DataflowProcessor proc, Object item ) { + protected processItem( DataflowProcessor dp, Object item ) { def value = closure ? closure.call(item) : item // when the value is a list, the first item hold the grouping key @@ -187,13 +187,13 @@ class CollectFileOp { * * @params obj: NOT USED. It needs to be declared because this method is invoked as a closure */ - protected emitItems(DataflowProcessor processor) { + protected emitItems(DataflowProcessor dp) { // emit collected files to 'result' channel collector.saveTo(storeDir).each { - Op.bind(processor, result, it) + Op.bind(dp, result, it) } // close the channel - Op.bind(processor, result, Channel.STOP) + Op.bind(dp, result, Channel.STOP) // close the collector collector.safeClose() } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/CollectOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/CollectOp.groovy index e7e3fd569a..f2fc3af815 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/CollectOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/CollectOp.groovy @@ -61,9 +61,9 @@ class CollectOp { .withSource(source) .withContext(new ContextGrouping()) .withOnNext { append(result, it) } - .withOnComplete { DataflowProcessor processor -> + .withOnComplete { DataflowProcessor dp -> final msg = result ? new ArrayBag(normalise(result)) : Channel.STOP - Op.bind(processor, target, msg) + Op.bind(dp, target, msg) } .apply() diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/CombineOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/CombineOp.groovy index 812b66ca03..139da19690 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/CombineOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/CombineOp.groovy @@ -99,7 +99,7 @@ class CombineOp { private Map handler(int index, DataflowWriteChannel target, AtomicInteger stopCount) { - def opts = new LinkedHashMap(2) + final opts = new LinkedHashMap(2) opts.onNext = { if( pivot ) { final pair = makeKey(pivot, it, context.getOperatorRun()) @@ -110,9 +110,9 @@ class CombineOp { } } - opts.onComplete = { DataflowProcessor proc -> + opts.onComplete = { DataflowProcessor dp -> if( stopCount.decrementAndGet()==0) { - Op.bind(proc, target, Channel.STOP) + Op.bind(dp, target, Channel.STOP) }} return opts diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/ConcatOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/ConcatOp.groovy index 83ad879c8f..a8a1159f57 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/ConcatOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/ConcatOp.groovy @@ -60,10 +60,10 @@ class ConcatOp { new SubscribeOp() .withSource(current) .withContext(context) - .withOnNext { DataflowProcessor proc, it -> Op.bind(proc, result, it) } - .withOnComplete { DataflowProcessor proc -> + .withOnNext { DataflowProcessor dp, Object it -> Op.bind(dp, result, it) } + .withOnComplete { DataflowProcessor dp -> if(next) append(result, channels, index) - else Op.bind(proc, result, Channel.STOP) + else Op.bind(dp, result, Channel.STOP) } .apply() } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy index 5a999183e9..503888992c 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy @@ -91,7 +91,7 @@ class DataflowHelper { @PackageScope static DEF_ERROR_LISTENER = new DataflowEventAdapter() { @Override - boolean onException(final DataflowProcessor processor, final Throwable t) { + boolean onException(final DataflowProcessor dp, final Throwable t) { final e = t instanceof InvocationTargetException ? t.cause : t OperatorImpl.log.error("@unknown", e) session?.abort(e) @@ -104,16 +104,16 @@ class DataflowHelper { new DataflowEventAdapter() { @Override - void afterRun(final DataflowProcessor processor, final List messages) { + void afterRun(final DataflowProcessor dp, final List messages) { if( source instanceof DataflowExpression ) { if( target !instanceof DataflowExpression ) - Op.bind(processor, target, Channel.STOP ) - processor.terminate() + Op.bind(dp, target, Channel.STOP ) + dp.terminate() } } @Override - boolean onException(final DataflowProcessor processor, final Throwable e) { + boolean onException(final DataflowProcessor dp, final Throwable e) { DataflowHelper.log.error("@unknown", e) session.abort(e) return true diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/FirstOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/FirstOp.groovy index f6dadb15a5..dfb9cfd729 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/FirstOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/FirstOp.groovy @@ -71,9 +71,9 @@ class FirstOp { final listener = new DataflowEventAdapter() { @Override - void afterStop(DataflowProcessor proc) { + void afterStop(DataflowProcessor dp) { if( stopOnFirst && !target.isBound() ) - Op.bind(proc, target, Channel.STOP) + Op.bind(dp, target, Channel.STOP) } } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/IntoOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/IntoOp.groovy index cfcb0390e1..75570b69ea 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/IntoOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/IntoOp.groovy @@ -107,10 +107,10 @@ class IntoOp { final stopOnFirst = source instanceof DataflowExpression final listener = new DataflowEventAdapter() { @Override - void afterRun(DataflowProcessor processor, List messages) { + void afterRun(DataflowProcessor dp, List messages) { if( !stopOnFirst ) return // -- terminate the process - processor.terminate() + dp.terminate() // -- close the output channels for( def it : outputs ) { if( !(it instanceof DataflowExpression)) @@ -123,7 +123,7 @@ class IntoOp { } @Override - public boolean onException(final DataflowProcessor processor, final Throwable e) { + public boolean onException(final DataflowProcessor dp, final Throwable e) { log.error("@unknown", e) session.abort(e) return true; diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/JoinOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/JoinOp.groovy index f2ca6a17b4..d3f0367342 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/JoinOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/JoinOp.groovy @@ -125,7 +125,7 @@ class JoinOp { final Map result = new HashMap<>(2) - result.onNext = { DataflowProcessor proc, Object it -> + result.onNext = { DataflowProcessor dp, Object it -> synchronized (this) { if(!failed) try { final entries = join0(buffer, size, index, it) @@ -135,12 +135,12 @@ class JoinOp { } catch (Exception e) { failed = true - Op.bind(proc, target, Channel.STOP) + Op.bind(dp, target, Channel.STOP) throw e } }} - result.onComplete = { DataflowProcessor proc -> + result.onComplete = { DataflowProcessor dp -> if( stopCount.decrementAndGet()==0 && !failed ) { try { if( remainder || failOnDuplicate ) @@ -149,7 +149,7 @@ class JoinOp { checkForMismatch(buffer) } finally { - Op.bind(proc, target, Channel.STOP) + Op.bind(dp, target, Channel.STOP) } }} diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/LastOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/LastOp.groovy index d7fd95631b..007cf7c392 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/LastOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/LastOp.groovy @@ -56,7 +56,7 @@ class LastOp { new SubscribeOp() .withSource(source) .withOnNext{ last = it } - .withOnComplete{ DataflowProcessor proc -> Op.bind(proc, target, last) } + .withOnComplete{ DataflowProcessor dp -> Op.bind(dp, target, last) } .apply() return target } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/MathOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/MathOp.groovy index 50e11bbc26..16eb1502a8 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/MathOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/MathOp.groovy @@ -80,8 +80,8 @@ class MathOp { return target } - private void completion(DataflowProcessor proc) { - Op.bind(proc, target, aggregate.result ) + private void completion(DataflowProcessor dp) { + Op.bind(dp, target, aggregate.result ) } /** diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/MixOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/MixOp.groovy index c9a0cf3f89..53b9e20a04 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/MixOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/MixOp.groovy @@ -68,8 +68,8 @@ class MixOp { target = CH.create() final count = new AtomicInteger( others.size()+1 ) final handlers = [ - onNext: { DataflowProcessor proc, it -> Op.bind(proc, target, it) }, - onComplete: { DataflowProcessor proc -> if(count.decrementAndGet()==0) { Op.bind(proc, target, Channel.STOP) } } + onNext: { DataflowProcessor dp, it -> Op.bind(dp, target, it) }, + onComplete: { DataflowProcessor dp -> if(count.decrementAndGet()==0) { Op.bind(dp, target, Channel.STOP) } } ] subscribe0(source, handlers) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy index a604c96d45..b1fad8eaee 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy @@ -555,17 +555,17 @@ class OperatorImpl { listeners << new DataflowEventAdapter() { @Override - void afterRun(final DataflowProcessor processor, final List messages) { + void afterRun(final DataflowProcessor dp, final List messages) { if( stopOnFirst ) - processor.terminate() + dp.terminate() } @Override - void afterStop(final DataflowProcessor processor) { - processor.bindOutput(Channel.STOP) + void afterStop(final DataflowProcessor dp) { + dp.bindOutput(Channel.STOP) } - boolean onException(final DataflowProcessor processor, final Throwable e) { + boolean onException(final DataflowProcessor dp, final Throwable e) { OperatorImpl.log.error("@unknown", e) session.abort(e) return true; @@ -657,7 +657,7 @@ class OperatorImpl { // -- intercepts the PoisonPill and sent out the items remaining in the buffer when the 'remainder' flag is true def listener = new DataflowEventAdapter() { - Object controlMessageArrived(final DataflowProcessor processor, final DataflowReadChannel channel, final int index, final Object message) { + Object controlMessageArrived(final DataflowProcessor dp, final DataflowReadChannel channel, final int index, final Object message) { if( message instanceof PoisonPill && keepRemainder && allBuffers.size() ) { allBuffers.each { target.bind( it ) @@ -668,7 +668,7 @@ class OperatorImpl { } @Override - boolean onException(DataflowProcessor processor, Throwable e) { + boolean onException(DataflowProcessor dp, Throwable e) { OperatorImpl.log.error("@unknown", e) session.abort(e) return true diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/ReduceOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/ReduceOp.groovy index decf065708..ff1441a1e8 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/ReduceOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/ReduceOp.groovy @@ -99,14 +99,14 @@ class ReduceOp { /* * when terminates bind the result value */ - void afterStop(final DataflowProcessor processor) { + void afterStop(final DataflowProcessor dp) { final result = beforeBind ? beforeBind.call(accum) : accum - Op.bind(processor, target, result) + Op.bind(dp, target, result) } - boolean onException(final DataflowProcessor processor, final Throwable e) { + boolean onException(final DataflowProcessor dp, final Throwable e) { log.error("@unknown", e) session.abort(e) return true; diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/SubscribeOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/SubscribeOp.groovy index 09d90b118b..1a91a76a30 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/SubscribeOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/SubscribeOp.groovy @@ -91,10 +91,10 @@ class SubscribeOp { def listener = new DataflowEventAdapter() { @Override - void afterStop(final DataflowProcessor processor) { + void afterStop(final DataflowProcessor dp) { if( !onComplete || error ) return try { - onComplete.call(processor) + onComplete.call(dp) } catch( Exception e ) { SubscribeOp.log.error("@unknown", e) @@ -103,7 +103,7 @@ class SubscribeOp { } @Override - boolean onException(final DataflowProcessor processor, final Throwable e) { + boolean onException(final DataflowProcessor dp, final Throwable e) { error = true if( !onError ) { log.error("@unknown", e) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/TakeOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/TakeOp.groovy index 0abe0c777c..5924c2042f 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/TakeOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/TakeOp.groovy @@ -57,14 +57,14 @@ class TakeOp { final listener = new DataflowEventAdapter() { @Override - void afterRun(final DataflowProcessor processor, final List messages) { + void afterRun(final DataflowProcessor dp, final List messages) { if( ++count >= length ) { - processor.bindOutput( Channel.STOP ) - processor.terminate() + dp.bindOutput( Channel.STOP ) + dp.terminate() } } - boolean onException(final DataflowProcessor processor, final Throwable e) { + boolean onException(final DataflowProcessor dp, final Throwable e) { TakeOp.log.error("@unknown", e) (Global.session as Session).abort(e) return true; diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/ToListOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/ToListOp.groovy index 4ad4e44589..5222a8fb40 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/ToListOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/ToListOp.groovy @@ -55,7 +55,7 @@ class ToListOp { new SubscribeOp() .withSource(source) .withOnNext({ result.add(it) }) - .withOnComplete({ DataflowProcessor processor -> Op.bind(processor, target, result) }) + .withOnComplete({ DataflowProcessor dp -> Op.bind(dp, target, result) }) .apply() return target } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/UniqueOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/UniqueOp.groovy index 1feac84192..d5d632e59f 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/UniqueOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/UniqueOp.groovy @@ -69,7 +69,7 @@ class UniqueOp { // when the operator stop clear the history map final listener = new DataflowEventAdapter() { - void afterStop(final DataflowProcessor processor) { + void afterStop(final DataflowProcessor dp) { history.clear() } } From 84f7806aa0eab80b7aa00008e631d38a1f642afd Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 14 Jan 2025 23:53:10 +0100 Subject: [PATCH 38/66] Add flatten operator [ci fast] Signed-off-by: Paolo Di Tommaso --- .../nextflow/extension/FlattenOp.groovy | 106 ++++++++++++++++++ .../nextflow/extension/OperatorImpl.groovy | 48 +------- .../test/groovy/nextflow/prov/ProvTest.groovy | 37 ++++++ 3 files changed, 146 insertions(+), 45 deletions(-) create mode 100644 modules/nextflow/src/main/groovy/nextflow/extension/FlattenOp.groovy diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/FlattenOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/FlattenOp.groovy new file mode 100644 index 0000000000..b7d629535b --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/extension/FlattenOp.groovy @@ -0,0 +1,106 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.extension + + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import groovyx.gpars.dataflow.DataflowReadChannel +import groovyx.gpars.dataflow.DataflowWriteChannel +import groovyx.gpars.dataflow.expression.DataflowExpression +import groovyx.gpars.dataflow.operator.DataflowEventAdapter +import groovyx.gpars.dataflow.operator.DataflowProcessor +import nextflow.Channel +import nextflow.Global +import nextflow.Session +import nextflow.extension.op.Op +/** + * Implements "flatten" operator + * + * @author Paolo Di Tommaso + */ +@Slf4j +@CompileStatic +class FlattenOp { + + private static Session getSession() { Global.getSession() as Session } + + private DataflowReadChannel source + private DataflowWriteChannel target + + FlattenOp withSource(DataflowReadChannel source) { + assert source!=null + this.source = source + return this + } + + FlattenOp setTarget( DataflowWriteChannel target ) { + this.target = target + return this + } + + DataflowWriteChannel apply() { + final target = CH.create() + final stopOnFirst = source instanceof DataflowExpression + + final listener = new DataflowEventAdapter() { + @Override + void afterRun(final DataflowProcessor dp, final List messages) { + if( stopOnFirst ) + dp.terminate() + } + + @Override + void afterStop(final DataflowProcessor dp) { + Op.bind(dp, target, Channel.STOP) + } + + boolean onException(final DataflowProcessor dp, final Throwable e) { + FlattenOp.log.error("@unknown", e) + session.abort(e) + return true; + } + } + + new Op() + .withInput(source) + .withListener(listener) + .withCode { Object item -> + final dp = getDelegate() as DataflowProcessor + switch( item ) { + case Collection: + ((Collection)item).flatten().each { value -> Op.bind(dp, target, value) } + break + + case (Object[]): + ((Collection)item).flatten().each { value -> Op.bind(dp, target, value) } + break + + case Channel.VOID: + break + + default: + Op.bind(dp, target, item) + } + } + .apply() + + return target + } + +} diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy index b1fad8eaee..c736a219a6 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy @@ -548,51 +548,9 @@ class OperatorImpl { } DataflowWriteChannel flatten( final DataflowReadChannel source ) { - - final listeners = [] - final target = CH.create() - final stopOnFirst = source instanceof DataflowExpression - - listeners << new DataflowEventAdapter() { - @Override - void afterRun(final DataflowProcessor dp, final List messages) { - if( stopOnFirst ) - dp.terminate() - } - - @Override - void afterStop(final DataflowProcessor dp) { - dp.bindOutput(Channel.STOP) - } - - boolean onException(final DataflowProcessor dp, final Throwable e) { - OperatorImpl.log.error("@unknown", e) - session.abort(e) - return true; - } - } - - newOperator(inputs: [source], outputs: [target], listeners: listeners) { item -> - - def proc = ((DataflowProcessor) getDelegate()) - switch( item ) { - case Collection: - ((Collection)item).flatten().each { value -> proc.bindOutput(value) } - break - - case (Object[]): - ((Collection)item).flatten().each { value -> proc.bindOutput(value) } - break - - case Channel.VOID: - break - - default: - proc.bindOutput(item) - } - } - - return target + new FlattenOp() + .withSource(source) + .apply() } /** diff --git a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy index 9e97dd3cb6..5058b374bb 100644 --- a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy @@ -735,4 +735,41 @@ class ProvTest extends Dsl2Spec { upstreamTasksOf('p3 (4)') .name == ['p2 (2)'] } + + def 'should track provenance with flatten operator' () { + when: + dsl_eval(globalConfig(), ''' + workflow { + channel.of([1,'a'], [2,'b']) \ + | p1 \ + | flatten \ + | p2 + } + + process p1 { + input: val(x) + output: val(y) + exec: + y = x + } + + process p2 { + input: val(x) + exec: + println x + } + ''') + then: + upstreamTasksOf('p2 (1)') + .name == ['p1 (1)'] + and: + upstreamTasksOf('p2 (2)') + .name == ['p1 (1)'] + and: + upstreamTasksOf('p2 (3)') + .name == ['p1 (2)'] + and: + upstreamTasksOf('p2 (4)') + .name == ['p1 (2)'] + } } From c139bec89a54b9250bb8e006793510f7c680113f Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 14 Jan 2025 23:59:06 +0100 Subject: [PATCH 39/66] Refactor Signed-off-by: Paolo Di Tommaso --- .../nextflow/extension/DataflowHelper.groovy | 7 ---- .../groovy/nextflow/extension/PhaseOp.groovy | 40 ++++++++++--------- 2 files changed, 21 insertions(+), 26 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy index 503888992c..5ed5951cdb 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy @@ -251,11 +251,4 @@ class DataflowHelper { } } - @CompileStatic - static Map eventsMap(Closure onNext, Closure onComplete) { - def result = new HashMap(2) - result.put('onNext', onNext) - result.put('onComplete', onComplete) - return result - } } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/PhaseOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/PhaseOp.groovy index 965a8247eb..dae8ea981d 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/PhaseOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/PhaseOp.groovy @@ -84,27 +84,30 @@ class PhaseOp { * @return A map with {@code OnNext} and {@code onComplete} methods entries */ static private final Map phaseHandler( Map> buffer, int size, int index, DataflowWriteChannel target, Closure mapper, AtomicInteger stopCount, boolean remainder ) { - - DataflowHelper.eventsMap( - { - synchronized (buffer) { - def entries = phaseImpl(buffer, size, index, it, mapper, false) - if( entries ) { - target.bind(entries) - } - }}, - - { - if( stopCount.decrementAndGet()==0) { - if( remainder ) - phaseRemainder(buffer,size, target) - target << Channel.STOP - }} - + eventsMap( + { + synchronized (buffer) { + def entries = phaseImpl(buffer, size, index, it, mapper, false) + if( entries ) { + target.bind(entries) + } + }}, + + { + if( stopCount.decrementAndGet()==0) { + if( remainder ) + phaseRemainder(buffer,size, target) + target << Channel.STOP + }} ) - } + static private Map eventsMap(Closure onNext, Closure onComplete) { + def result = new HashMap(2) + result.put('onNext', onNext) + result.put('onComplete', onComplete) + return result + } /** * Implements the phase operator logic. Basically buffers the values received on each channel by their key . @@ -184,7 +187,6 @@ class PhaseOp { return result } - static private final void phaseRemainder( Map> buffers, int count, DataflowWriteChannel target ) { Collection> slots = buffers.values() From 8f8ff02946db4cc223a50b62b28e510bfc6cf241 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Wed, 15 Jan 2025 00:02:20 +0100 Subject: [PATCH 40/66] Minor change Signed-off-by: Paolo Di Tommaso --- .../groovy/nextflow/extension/IntoOp.groovy | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/IntoOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/IntoOp.groovy index 75570b69ea..17de3dac12 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/IntoOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/IntoOp.groovy @@ -46,11 +46,9 @@ class IntoOp { private Session session = (Session)Global.session - IntoOp( DataflowReadChannel source, List targets ) { assert source assert targets - this.source = source this.outputs = targets } @@ -67,27 +65,6 @@ class IntoOp { this.outputs = targets } - IntoOp( DataflowReadChannel source, Closure holder ) { - assert source - assert holder - - final names = CaptureProperties.capture(holder) - if( !names ) - throw new IllegalArgumentException("Missing target channel names in `into` operator") - if( names.size() == 1 ) - log.warn("The `into` operator should be used to connect two or more target channels -- consider replacing it with `.set { ${names[0]} }`") - - List targets = [] - names.each { identifier -> - def channel = newChannelBy(source) - targets.add(channel) - NF.binding.setVariable(identifier, channel) - } - - this.source = source - this.outputs = targets - } - List getOutputs() { outputs } IntoOp apply() { From 702705f9d51c6e53f0d4fdaaea9f270a01c9c4f2 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Wed, 15 Jan 2025 14:18:25 +0100 Subject: [PATCH 41/66] Add sum and mean operators Signed-off-by: Paolo Di Tommaso --- .../groovy/nextflow/extension/MathOp.groovy | 5 +- .../test/groovy/nextflow/prov/ProvTest.groovy | 83 ++++++++++++++++++- 2 files changed, 85 insertions(+), 3 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/MathOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/MathOp.groovy index 16eb1502a8..ec7792c59c 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/MathOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/MathOp.groovy @@ -25,6 +25,7 @@ import groovyx.gpars.dataflow.DataflowVariable import groovyx.gpars.dataflow.DataflowWriteChannel import groovyx.gpars.dataflow.operator.DataflowProcessor import nextflow.Channel +import nextflow.extension.op.ContextGrouping import nextflow.extension.op.Op /** * Implements the logic for "sum" and "mean" operators @@ -75,7 +76,9 @@ class MathOp { target = new DataflowVariable() new SubscribeOp() .withSource(source) - .withEvents(onNext: aggregate.&process, onComplete: this.&completion) + .withContext(new ContextGrouping()) + .withOnNext(aggregate.&process) + .withOnComplete(this.&completion) .apply() return target } diff --git a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy index 5058b374bb..1f39f1eeb7 100644 --- a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy @@ -311,7 +311,7 @@ class ProvTest extends Dsl2Spec { } - def 'should track provenance with distinc operator'() { + def 'should track provenance with distinct operator'() { when: dsl_eval(globalConfig(), ''' @@ -507,7 +507,7 @@ class ProvTest extends Dsl2Spec { .name == ['p1 (1)', 'p1 (2)', 'p1 (3)'] } - @Ignore // this should be review + @Ignore // the semantic of this should be reviewed def 'should track provenance with min operator'() { when: dsl_eval(globalConfig(), ''' @@ -534,6 +534,85 @@ class ProvTest extends Dsl2Spec { .name == ['p1 (3)'] } + @Ignore // the semantic of this should be reviewed + def 'should track provenance with max operator'() { + when: + dsl_eval(globalConfig(), ''' + workflow { + channel.of(3,2,1) | p1 | max | p2 + } + + process p1 { + input: val(x) + output: val(y) + exec: + y = x + } + + process p2 { + input: val(x) + exec: + println x + } + ''') + + then: + upstreamTasksOf('p2') + .name == ['p1 (1)'] + } + + def 'should track provenance with sum operator'() { + when: + dsl_eval(globalConfig(), ''' + workflow { + channel.of(3,2,1) | p1 | sum | p2 + } + + process p1 { + input: val(x) + output: val(y) + exec: + y = x + } + + process p2 { + input: val(x) + exec: + println x + } + ''') + + then: + upstreamTasksOf('p2') + .name == ['p1 (1)', 'p1 (2)', 'p1 (3)'] + } + + def 'should track provenance with mean operator'() { + when: + dsl_eval(globalConfig(), ''' + workflow { + channel.of(3,2,1) | p1 | mean | p2 + } + + process p1 { + input: val(x) + output: val(y) + exec: + y = x + } + + process p2 { + input: val(x) + exec: + println x + } + ''') + + then: + upstreamTasksOf('p2') + .name == ['p1 (1)', 'p1 (2)', 'p1 (3)'] + } + def 'should track provenance with buffer operator'() { when: dsl_eval(globalConfig(), ''' From 5750f718c902a3700452506154b39840358c92de Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Wed, 15 Jan 2025 16:17:05 +0100 Subject: [PATCH 42/66] Add multimap operator [ci fast] Signed-off-by: Paolo Di Tommaso --- .../nextflow/extension/MultiMapOp.groovy | 12 ++-- .../test/groovy/nextflow/prov/ProvTest.groovy | 56 ++++++++++++++++++- 2 files changed, 62 insertions(+), 6 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/MultiMapOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/MultiMapOp.groovy index 467c734e09..875ca1da63 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/MultiMapOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/MultiMapOp.groovy @@ -20,7 +20,9 @@ import groovy.transform.CompileStatic import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowWriteChannel import groovyx.gpars.dataflow.expression.DataflowExpression +import groovyx.gpars.dataflow.operator.DataflowProcessor import nextflow.Channel +import nextflow.extension.op.Op import nextflow.script.ChannelOut import nextflow.script.TokenMultiMapDef /** @@ -48,20 +50,20 @@ class MultiMapOp { ChannelOut getOutput() { this.output } - protected void doNext(it) { + protected void doNext(DataflowProcessor dp, Object it) { final ret = (Map)forkDef.closure.call(it) for( Map.Entry entry : ret.entrySet() ) { - targets[entry.key].bind(entry.value) + Op.bind(dp, targets[entry.key], entry.value) } } - protected void doComplete(nope) { + protected void doComplete(DataflowProcessor dp) { for( DataflowWriteChannel ch : targets.values() ) { if( ch instanceof DataflowExpression ) { - if( !ch.isBound()) ch.bind(Channel.STOP) + if( !ch.isBound()) Op.bind(dp, ch, Channel.STOP) } else { - ch.bind(Channel.STOP) + Op.bind(dp, ch, Channel.STOP) } } } diff --git a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy index 1f39f1eeb7..4c5311355b 100644 --- a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy @@ -53,8 +53,62 @@ class ProvTest extends Dsl2Spec { .name == ['p1'] } - def 'should track provenance with branch operator'() { + def 'should track provenance with multiMap operator'() { + when: + dsl_eval(globalConfig(), ''' + workflow { + channel.of(1,2,3) | p1 + + p1.out.multiMap { v -> + foo: v + 1 + bar: v * v + } + .set { result } + + result.foo | p2 + result.bar | p3 + } + + process p1 { + input: val(x) + output: val(y) + exec: + y = x + } + + process p2 { + input: val(x) + exec: + println x + } + + process p3 { + input: val(x) + exec: + println x + } + ''') + then: + upstreamTasksOf('p2 (1)') + .name == ['p1 (1)'] + and: + upstreamTasksOf('p2 (2)') + .name == ['p1 (2)'] + and: + upstreamTasksOf('p2 (3)') + .name == ['p1 (3)'] + and: + upstreamTasksOf('p3 (1)') + .name == ['p1 (1)'] + and: + upstreamTasksOf('p3 (2)') + .name == ['p1 (2)'] + and: + upstreamTasksOf('p3 (3)') + .name == ['p1 (3)'] + } + def 'should track provenance with branch operator'() { when: dsl_eval(globalConfig(), ''' workflow { From 14484a50425b2cf1a011b17a5ffe867f3fd0a9c5 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Wed, 15 Jan 2025 19:45:46 +0100 Subject: [PATCH 43/66] Fix tests [ci fast] Signed-off-by: Paolo Di Tommaso --- .../nextflow/extension/MultiMapOpTest.groovy | 61 +++++++++---------- 1 file changed, 30 insertions(+), 31 deletions(-) diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/MultiMapOpTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/MultiMapOpTest.groovy index 75dbb611f2..1b931886a1 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/MultiMapOpTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/MultiMapOpTest.groovy @@ -16,6 +16,7 @@ package nextflow.extension +import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowVariable import org.junit.Rule @@ -43,24 +44,24 @@ class MultiMapOpTest extends Dsl2Spec { bar: it*it+2 baz: 3 } - ''') + ''') as List then: result.size() == 3 and: - result[0].val == 1 - result[0].val == 2 - result[0].val == 3 - result[0].val == Channel.STOP + result[0].unwrap() == 1 + result[0].unwrap() == 2 + result[0].unwrap() == 3 + result[0].unwrap() == Channel.STOP and: - result[1].val == 2 - result[1].val == 3 - result[1].val == 6 - result[1].val == Channel.STOP + result[1].unwrap() == 2 + result[1].unwrap() == 3 + result[1].unwrap() == 6 + result[1].unwrap() == Channel.STOP and: - result[2].val == 3 - result[2].val == 3 - result[2].val == 3 - result[2].val == Channel.STOP + result[2].unwrap() == 3 + result[2].unwrap() == 3 + result[2].unwrap() == 3 + result[2].unwrap() == Channel.STOP } @@ -75,25 +76,24 @@ class MultiMapOpTest extends Dsl2Spec { bar: p*p+2 baz: p-1 } - ''') + ''') as List then: result.size() == 3 and: - result[0].val == 1 - result[0].val == 2 - result[0].val == 3 - result[0].val == Channel.STOP + result[0].unwrap() == 1 + result[0].unwrap() == 2 + result[0].unwrap() == 3 + result[0].unwrap() == Channel.STOP and: - result[1].val == 2 - result[1].val == 3 - result[1].val == 6 - result[1].val == Channel.STOP + result[1].unwrap() == 2 + result[1].unwrap() == 3 + result[1].unwrap() == 6 + result[1].unwrap() == Channel.STOP and: - result[2].val == -1 - result[2].val == 0 - result[2].val == 1 - result[2].val == Channel.STOP - + result[2].unwrap() == -1 + result[2].unwrap() == 0 + result[2].unwrap() == 1 + result[2].unwrap() == Channel.STOP } def 'should pass criteria as argument' () { @@ -131,16 +131,15 @@ class MultiMapOpTest extends Dsl2Spec { foo: p.toUpperCase() bar: p.reverse() } - ''') + ''') as List then: result.size() == 2 and: result[0] instanceof DataflowVariable - result[0].val == 'HELLO' + result[0].unwrap() == 'HELLO' and: result[1] instanceof DataflowVariable - result[1].val == 'olleh' - + result[1].unwrap() == 'olleh' } } From ff74318a1df0be4f5ef11b1111164b9a1008d67c Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Wed, 15 Jan 2025 20:26:59 +0100 Subject: [PATCH 44/66] Minor [ci fast] Signed-off-by: Paolo Di Tommaso --- .../groovy/nextflow/extension/BufferOp.groovy | 16 ++++++++-------- .../groovy/nextflow/extension/CombineOp.groovy | 2 +- .../groovy/nextflow/extension/DistinctOp.groovy | 4 ++-- .../groovy/nextflow/extension/FilterOp.groovy | 6 +++--- .../groovy/nextflow/extension/FirstOp.groovy | 6 +++--- .../groovy/nextflow/extension/FlatMapOp.groovy | 14 +++++++------- .../main/groovy/nextflow/extension/op/Op.groovy | 10 +++++----- 7 files changed, 29 insertions(+), 29 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/BufferOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/BufferOp.groovy index bfa5f41bd2..0a18688962 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/BufferOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/BufferOp.groovy @@ -164,16 +164,16 @@ class BufferOp { final listener = new DataflowEventAdapter() { @Override - Object controlMessageArrived(final DataflowProcessor processor, final DataflowReadChannel channel, final int index, final Object message) { + Object controlMessageArrived(final DataflowProcessor dp, final DataflowReadChannel channel, final int index, final Object message) { if( message instanceof PoisonPill && remainder && buffer.size() ) { - Op.bind(processor,target, buffer) + Op.bind(dp, target, buffer) } return message } @Override - void afterStop(DataflowProcessor processor) { - Op.bind(processor, target, Channel.STOP) + void afterStop(DataflowProcessor dp) { + Op.bind(dp, target, Channel.STOP) } @Override @@ -196,17 +196,17 @@ class BufferOp { isOpen = true buffer << it } - final proc = getDelegate() as DataflowProcessor + final dp = getDelegate() as DataflowProcessor if( closeCriteria.call(it) ) { - Op.bind(proc, target, buffer) + Op.bind(dp, target, buffer) buffer = [] // when a *startingCriteria* is defined, close the open frame flag isOpen = (startingCriteria == null) } if( stopOnFirst ) { if( remainder && buffer ) - Op.bind(proc, target, buffer) - proc.terminate() + Op.bind(dp, target, buffer) + dp.terminate() } } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/CombineOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/CombineOp.groovy index 139da19690..3977711e12 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/CombineOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/CombineOp.groovy @@ -111,7 +111,7 @@ class CombineOp { } opts.onComplete = { DataflowProcessor dp -> - if( stopCount.decrementAndGet()==0) { + if( stopCount.decrementAndGet()==0 ) { Op.bind(dp, target, Channel.STOP) }} diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/DistinctOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/DistinctOp.groovy index fde51bc16b..00df03a9d1 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/DistinctOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/DistinctOp.groovy @@ -61,11 +61,11 @@ class DistinctOp { def previous = null final code = { - final proc = getDelegate() as DataflowProcessor + final dp = getDelegate() as DataflowProcessor final key = comparator.call(it) if( key != previous ) { previous = key - Op.bind(proc, target, it) + Op.bind(dp, target, it) } } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/FilterOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/FilterOp.groovy index e3250216bc..df5bec5842 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/FilterOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/FilterOp.groovy @@ -68,12 +68,12 @@ class FilterOp { final result = criteria instanceof Closure ? DefaultTypeTransformation.castToBoolean(criteria.call(it)) : discriminator.invoke(criteria, (Object)it) - final proc = getDelegate() as DataflowProcessor + final dp = getDelegate() as DataflowProcessor if( result ) { - Op.bind(proc, target, it) + Op.bind(dp, target, it) } if( stopOnFirst ) { - Op.bind(proc, target, Channel.STOP) + Op.bind(dp, target, Channel.STOP) } }) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/FirstOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/FirstOp.groovy index dfb9cfd729..22805ad527 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/FirstOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/FirstOp.groovy @@ -78,12 +78,12 @@ class FirstOp { } final code = { - final proc = getDelegate() as DataflowProcessor + final dp = getDelegate() as DataflowProcessor final accept = discriminator.invoke(criteria, it) if( accept ) - Op.bind(proc, target, it) + Op.bind(dp, target, it) if( accept || stopOnFirst ) - proc.terminate() + dp.terminate() } new Op() diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/FlatMapOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/FlatMapOp.groovy index 79d4f1fe75..86959732de 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/FlatMapOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/FlatMapOp.groovy @@ -65,31 +65,31 @@ class FlatMapOp { .withListener(stopErrorListener(source,target)) .withCode { Object item -> final result = mapper != null ? mapper.call(item) : item - final proc = getDelegate() as DataflowProcessor + final dp = getDelegate() as DataflowProcessor switch( result ) { case Collection: - result.each { it -> Op.bind(proc, target,it) } + result.each { it -> Op.bind(dp, target,it) } break case (Object[]): - result.each { it -> Op.bind(proc, target,it) } + result.each { it -> Op.bind(dp, target,it) } break case Map: - result.each { it -> Op.bind(proc, target,it) } + result.each { it -> Op.bind(dp, target,it) } break case Map.Entry: - Op.bind(proc, target, (result as Map.Entry).key ) - Op.bind(proc, target, (result as Map.Entry).value ) + Op.bind(dp, target, (result as Map.Entry).key ) + Op.bind(dp, target, (result as Map.Entry).value ) break case Channel.VOID: break default: - Op.bind(proc, target, result) + Op.bind(dp, target, result) } } .apply() diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy index f7ada40b30..6001b26099 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy @@ -61,16 +61,16 @@ class Op { obj instanceof Tracker.Msg ? obj : Tracker.Msg.of(obj) } - static void bind(DataflowProcessor operator, DataflowWriteChannel channel, Object msg) { + static void bind(DataflowProcessor dp, DataflowWriteChannel channel, Object msg) { try { if( msg instanceof PoisonPill ) { channel.bind(msg) - allContexts.remove(operator) + allContexts.remove(dp) } else { - final ctx = allContexts.get(operator) + final ctx = allContexts.get(dp) if( !ctx ) - throw new IllegalStateException("Cannot find any context for operator=$operator") + throw new IllegalStateException("Cannot find any context for operator=$dp") final run = ctx.getOperatorRun() Prov.getTracker().bindOutput(run, channel, msg) } @@ -81,7 +81,7 @@ class Op { } } - static bind(OperatorRun run, DataflowWriteChannel channel, Object msg) { + static void bind(OperatorRun run, DataflowWriteChannel channel, Object msg) { Prov.getTracker().bindOutput(run, channel, msg) } From 9e5bc10f7c32f4cde6ecc90f064c7293e5dfd51f Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Wed, 15 Jan 2025 23:03:30 +0100 Subject: [PATCH 45/66] Add collate operator Signed-off-by: Paolo Di Tommaso --- .../nextflow/extension/CollateOp.groovy | 138 ++++++++++++++++++ .../groovy/nextflow/extension/JoinOp.groovy | 21 +-- .../nextflow/extension/OperatorImpl.groovy | 79 ++-------- .../groovy/nextflow/extension/op/Op.groovy | 16 ++ .../extension/OperatorImplTest.groovy | 2 +- .../test/groovy/nextflow/prov/ProvTest.groovy | 45 ++++++ 6 files changed, 213 insertions(+), 88 deletions(-) create mode 100644 modules/nextflow/src/main/groovy/nextflow/extension/CollateOp.groovy diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/CollateOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/CollateOp.groovy new file mode 100644 index 0000000000..307d8cbf6d --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/extension/CollateOp.groovy @@ -0,0 +1,138 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.extension + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import groovyx.gpars.dataflow.DataflowReadChannel +import groovyx.gpars.dataflow.DataflowWriteChannel +import groovyx.gpars.dataflow.operator.DataflowEventAdapter +import groovyx.gpars.dataflow.operator.DataflowProcessor +import groovyx.gpars.dataflow.operator.PoisonPill +import nextflow.Global +import nextflow.Session +import nextflow.extension.op.ContextRunPerThread +import nextflow.extension.op.Op +import nextflow.extension.op.OpContext +import nextflow.extension.op.OpDatum +/** + * Implement "collate" operator + * + * @author Paolo Di Tommaso + */ +@Slf4j +@CompileStatic +class CollateOp { + private DataflowReadChannel source + private DataflowWriteChannel target + private int size + private int step + private boolean remainder + private OpContext context = new ContextRunPerThread() + + private static Session getSession() { Global.getSession() as Session } + + CollateOp withSource(DataflowReadChannel source) { + assert source!=null + this.source = source + return this + } + + CollateOp withTarget( DataflowWriteChannel target ) { + this.target = target + return this + } + + CollateOp withSize(int size) { + this.size = size + return this + } + + CollateOp withStep(int step) { + this.step = step + return this + } + + CollateOp withRemainder(boolean keepRemainder) { + this.remainder = keepRemainder + return this + } + + DataflowWriteChannel apply() { + if( size <= 0 ) { + throw new IllegalArgumentException("Illegal argument 'size' for operator 'collate' -- it must be greater than zero: $size") + } + + if( step <= 0 ) { + throw new IllegalArgumentException("Illegal argument 'step' for operator 'collate' -- it must be greater than zero: $step") + } + + // the result queue + final target = CH.create() + + // the list holding temporary collected elements + List> allBuffers = [] + + // -- intercepts the PoisonPill and sent out the items remaining in the buffer when the 'remainder' flag is true + final listener = new DataflowEventAdapter() { + Object controlMessageArrived(final DataflowProcessor dp, final DataflowReadChannel channel, final int index, final Object message) { + if( message instanceof PoisonPill && remainder && allBuffers.size() ) { + for(List it : allBuffers) { + Op.bindRunValues(target, it, false) + } + } + return message + } + + @Override + boolean onException(DataflowProcessor dp, Throwable e) { + CollateOp.log.error("@unknown", e) + session.abort(e) + return true + } + } + + int index = 0 + new Op() + .withInput(source) + .withOutput(target) + .withContext(context) + .withListener(listener) + .withCode { + + if( index++ % step == 0 ) { + allBuffers.add( [] ) + } + + final run = context.getOperatorRun() + for( List list : allBuffers ) { + list.add(OpDatum.of(it,run)) + } + + final buf = allBuffers.head() + if( buf.size() == size ) { + Op.bindRunValues(target, buf, false) + allBuffers = allBuffers.tail() + } + } + .apply() + + return target + } + +} diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/JoinOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/JoinOp.groovy index d3f0367342..396ef3a883 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/JoinOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/JoinOp.groovy @@ -33,7 +33,6 @@ import nextflow.extension.op.ContextRunPerThread import nextflow.extension.op.Op import nextflow.extension.op.OpContext import nextflow.extension.op.OpDatum -import nextflow.prov.OperatorRun import nextflow.util.CheckHelper /** * Implements {@link OperatorImpl#join} operator logic @@ -130,7 +129,7 @@ class JoinOp { if(!failed) try { final entries = join0(buffer, size, index, it) if( entries ) { - emitEntries(target, entries) + Op.bindRunValues(target, entries, true) } } catch (Exception e) { @@ -156,20 +155,7 @@ class JoinOp { return result } - private void emitEntries(DataflowWriteChannel target, List entries) { - final inputs = new ArrayList(entries.size()) - final values = new ArrayList(entries.size()) - for( Object it : entries ) { - if( it instanceof OpDatum ) { - inputs.addAll(it.run.inputIds) - values.add(it.value) - } - else - values.add(it) - } - final run = new OperatorRun(inputs) - Op.bind(run, target, values.size()==1 ? values[0] : values) - } + /** * Implements the join operator logic. Basically buffers the values received on each channel by their key . @@ -278,12 +264,11 @@ class JoinOp { if( fill ) { // bind value to target channel if( remainder ) - emitEntries(target, result) + Op.bindRunValues(target, result, true) } else break } - } } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy index c736a219a6..97fa34a405 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy @@ -28,9 +28,6 @@ import groovyx.gpars.dataflow.DataflowVariable import groovyx.gpars.dataflow.DataflowWriteChannel import groovyx.gpars.dataflow.expression.DataflowExpression import groovyx.gpars.dataflow.operator.ChainWithClosure -import groovyx.gpars.dataflow.operator.DataflowEventAdapter -import groovyx.gpars.dataflow.operator.DataflowProcessor -import groovyx.gpars.dataflow.operator.PoisonPill import nextflow.Channel import nextflow.Global import nextflow.NF @@ -590,73 +587,20 @@ class OperatorImpl { throw new IllegalArgumentException("Illegal argument 'size' for operator 'collate' -- it must be greater than zero: $size") } - def target = new BufferOp(source) - .setParams( size: size, remainder: keepRemainder ) - .apply() - - return target + return new BufferOp(source) + .setParams( size: size, remainder: keepRemainder ) + .apply() } DataflowWriteChannel collate( DataflowReadChannel source, int size, int step, boolean keepRemainder = true ) { - if( size <= 0 ) { - throw new IllegalArgumentException("Illegal argument 'size' for operator 'collate' -- it must be greater than zero: $size") - } - - if( step <= 0 ) { - throw new IllegalArgumentException("Illegal argument 'step' for operator 'collate' -- it must be greater than zero: $step") - } - - // the result queue - final target = CH.create() - - // the list holding temporary collected elements - List> allBuffers = [] - - // -- intercepts the PoisonPill and sent out the items remaining in the buffer when the 'remainder' flag is true - def listener = new DataflowEventAdapter() { - - Object controlMessageArrived(final DataflowProcessor dp, final DataflowReadChannel channel, final int index, final Object message) { - if( message instanceof PoisonPill && keepRemainder && allBuffers.size() ) { - allBuffers.each { - target.bind( it ) - } - } - - return message; - } - - @Override - boolean onException(DataflowProcessor dp, Throwable e) { - OperatorImpl.log.error("@unknown", e) - session.abort(e) - return true - } - } - - - int index = 0 - - // -- the operator collecting the elements - newOperator( inputs: [source], outputs: [target], listeners: [listener]) { - - if( index++ % step == 0 ) { - allBuffers.add( [] ) - } - - allBuffers.each { List list -> list.add(it) } - - def buf = allBuffers.head() - if( buf.size() == size ) { - ((DataflowProcessor) getDelegate()).bindOutput(buf) - allBuffers = allBuffers.tail() - } - - } - - return target + new CollateOp() + .withSource(source) + .withSize(size) + .withStep(step) + .withRemainder(keepRemainder) + .apply() } - /** * Similar to https://github.com/Netflix/RxJava/wiki/Combining-Observables#merge * @@ -744,12 +688,9 @@ class OperatorImpl { } DataflowWriteChannel cross( DataflowReadChannel source, DataflowReadChannel other, Closure mapper = null ) { - - def target = new CrossOp(source, other) + return new CrossOp(source, other) .setMapper(mapper) .apply() - - return target } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy index 6001b26099..29ada71098 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy @@ -81,6 +81,22 @@ class Op { } } + static void bindRunValues(DataflowWriteChannel target, List entries, boolean singleton) { + final inputs = new ArrayList(entries.size()) + final values = new ArrayList(entries.size()) + for( Object it : entries ) { + if( it instanceof OpDatum ) { + inputs.addAll(it.run.inputIds) + values.add(it.value) + } + else + values.add(it) + } + final run = new OperatorRun(inputs) + final out = singleton && values.size()==1 ? values[0] : values + Op.bind(run, target, out) + } + static void bind(OperatorRun run, DataflowWriteChannel channel, Object msg) { Prov.getTracker().bindOutput(run, channel, msg) } diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/OperatorImplTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/OperatorImplTest.groovy index 85f3380e9f..29b8904088 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/OperatorImplTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/OperatorImplTest.groovy @@ -538,7 +538,7 @@ class OperatorImplTest extends Specification { r2.unwrap() == Channel.STOP when: - def r3 = Channel.of(1,2,3,4).collate( 3, 1 ) + def r3 = Channel.of(1,2,3,4).collate( 3, 1 ) then: r3.unwrap() == [1, 2, 3] r3.unwrap() == [2, 3, 4] diff --git a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy index 4c5311355b..4c1b1f5ead 100644 --- a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy @@ -509,6 +509,51 @@ class ProvTest extends Dsl2Spec { .name == ['p1 (1)', 'p1 (2)', 'p1 (3)'] } + def 'should track provenance with collate operator'() { + when: + dsl_eval(globalConfig(), ''' + workflow { + channel.of(1, 2, 3, 4) | p1 | collate( 3, 1 ) | p2 + } + + // + // the "collate" emits the following values: + // + // [1, 2, 3] + // [2, 3, 4] + // [3, 4] + // [4] + // + + process p1 { + input: val(x) + output: val(y) + exec: + y = x + } + + process p2 { + input: val(x) + exec: + println "$task.process ($task.index) = ${x}" + } + ''') + + then: + upstreamTasksOf('p2 (1)') + .name == ['p1 (1)', 'p1 (2)', 'p1 (3)' ] + and: + upstreamTasksOf('p2 (2)') + .name == ['p1 (2)', 'p1 (3)', 'p1 (4)'] + and: + upstreamTasksOf('p2 (3)') + .name == ['p1 (3)', 'p1 (4)'] + and: + upstreamTasksOf('p2 (4)') + .name == ['p1 (4)'] + + } + def 'should track provenance with count value operator'() { when: dsl_eval(globalConfig(), ''' From 6b926f989bfad08df15cd944fc737ba0f628fcc3 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sat, 18 Jan 2025 11:14:29 +0100 Subject: [PATCH 46/66] Update docs [ci skip] Signed-off-by: Paolo Di Tommaso --- .../groovy/nextflow/extension/op/Op.groovy | 114 +++++++++++++++++- .../nextflow/extension/op/OpClosure.groovy | 5 +- 2 files changed, 113 insertions(+), 6 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy index 29ada71098..d7c8462525 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy @@ -35,7 +35,7 @@ import nextflow.prov.OperatorRun import nextflow.prov.Prov import nextflow.prov.Tracker /** - * Operator helpers methods + * Model a dataflow operator * * @author Paolo Di Tommaso */ @@ -43,6 +43,10 @@ import nextflow.prov.Tracker @CompileStatic class Op { + /** + * Associate a {@link DataflowProcessor} with the corresponding {@link OpContext} object + * tracking running operator context + */ static final public ConcurrentHashMap allContexts = new ConcurrentHashMap<>(); static List unwrap(List messages) { @@ -81,6 +85,21 @@ class Op { } } + /** + * This method gets a list of one or more {@link OpDatum} objects, and binds + * the corresponding values to the specified channel using a new {@link OperatorRun} + * instance using as {@link OperatorRun#inputIds} the set of IDs obtained by the + * provide "entries" lists. + * + * @param target + * The dataflow channel where outputs are going to be bound. + * @param entries + * A list of one or more {@link OpDatum} holding the value to be bound. + * @param singleton + * When {@code true} and the entries list is provided, the value object is emitted + * in place of a list with a sole object, when {@code false} the list of values + * is emitted in any case. + */ static void bindRunValues(DataflowWriteChannel target, List entries, boolean singleton) { final inputs = new ArrayList(entries.size()) final values = new ArrayList(entries.size()) @@ -115,42 +134,99 @@ class Op { OpContext getContext() { context } Closure getCode() { code } + /** + * Set the operator input channel + * + * @param channel + * The channel from which the operator is receiving the input data. + * @return + * The operator object itself. + */ Op withInput(DataflowReadChannel channel) { assert channel != null this.inputs = List.of(channel) return this } + /** + * Set the operator input channels + * + * @param channels + * The channels from which the operator is receiving the input data + * @return + * The operator object itself. + */ Op withInputs(List channels) { assert channels != null this.inputs = channels return this } + /** + * Set the operator output channel + * + * @param channel + * The channel to which the operator is binding output data. + * @return + * The operator object itself. + */ Op withOutput(DataflowWriteChannel channel) { assert channel != null this.outputs = List.of(channel) return this } + /** + * Set the operator output channels + * + * @param channels + * The channels to which the operator is binding output data. + * @return + * The operator object itself. + */ Op withOutputs(List channels) { assert channels != null this.outputs = channels return this } + /** + * Set a {@link DataflowEventListener} associated with the operator. + * + * @param listener + * The {@link DataflowEventListener} listener object. + * @return + * The operator object itself. + */ Op withListener(DataflowEventListener listener) { if( listener ) this.listeners = List.of(listener) return this } + /** + * Set one or more {@link DataflowEventListener} objects associated with the operator. + * + * @param listeners + * The list of {@link DataflowEventListener} listener objects. + * @return + * The operator object itself. + */ Op withListeners(List listeners) { if( listeners ) this.listeners = listeners return this } + /** + * Set the operator inputs, outputs and listeners parameters using a map object. + * + * @param params + * A {@link Map} object holding the {@code inputs} and {@code outputs} channels + * and {@code listeners} objects. + * @return + * The operator object itself. + */ Op withParams(Map params) { if( params.inputs ) this.inputs = params.inputs as List @@ -161,18 +237,43 @@ class Op { return this } + /** + * Set the context object associated associated with the operator run. + * + * @param context + * The {@link OpContext} object associated with the operator execution. + * @return + * The operator object itself. + */ Op withContext(OpContext context) { if( context!=null ) this.context = context return this } + /** + * Set the operator action as a closure + * + * @param code + * The action that should be performed when the operator run. + * @return + * The operator object itself. + * + */ Op withCode(Closure code) { this.code = code return this } - Map toMap() { + /** + * Map the operator inputs, outputs and listeners to a Java {@Link Map} + * for compatibility with {@link DataflowProcessor} API + * + * @return + * A map object holding the operator {@code inputs}, {@code outputs} channels + * and the {@code listeners} object. + */ + protected Map toMap() { final ret = new HashMap() ret.inputs = inputs ?: List.of() ret.outputs = outputs ?: List.of() @@ -180,12 +281,19 @@ class Op { return ret } + /** + * Creates the {@link DataflowOperator} object and start it + * + * @return + * The corresponding {@link DataflowOperator} object + */ DataflowProcessor apply() { assert inputs assert code assert context - // create the underlying dataflow operator + // Encapsulate the target "code" closure with a "OpClosure" object + // to grab input data and track the execution provenance final closure = new OpClosure(code, context) final group = Dataflow.retrieveCurrentDFPGroup() final operator = new DataflowOperator(group, toMap(), closure) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/op/OpClosure.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/op/OpClosure.groovy index 16c359413d..77446ec6b3 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/op/OpClosure.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/op/OpClosure.groovy @@ -17,15 +17,14 @@ package nextflow.extension.op - import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import nextflow.prov.OperatorRun import nextflow.prov.Prov import org.codehaus.groovy.runtime.InvokerHelper /** - * A closure that wraps the execution of an operator target code (closure) - * and maps the inputs and outputs to the corresponding operator run. + * A closure that wraps the execution of an operator target code + * associating the inputs and outputs to the corresponding operator run. * * @author Paolo Di Tommaso */ From 8637aee8a1d353375545ddd2fb6245ec505b8943 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sat, 18 Jan 2025 11:23:50 +0100 Subject: [PATCH 47/66] Use a set object in place of list for operator inputs [ci fast] Signed-off-by: Paolo Di Tommaso --- .../src/main/groovy/nextflow/extension/CombineOp.groovy | 4 ++-- .../nextflow/src/main/groovy/nextflow/extension/op/Op.groovy | 4 ++-- .../src/main/groovy/nextflow/extension/op/OpDatum.groovy | 4 +++- .../nextflow/src/main/groovy/nextflow/prov/OperatorRun.groovy | 2 +- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/CombineOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/CombineOp.groovy index 3977711e12..2f3358632a 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/CombineOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/CombineOp.groovy @@ -157,9 +157,9 @@ class CombineOp { } private void bindValues(List p, a, b) { - final i = new ArrayList() + final i = new ArrayList() final t = tuple(p, OpDatum.unwrap(a,i), OpDatum.unwrap(b,i)) - final r = new OperatorRun(i) + final r = new OperatorRun(new LinkedHashSet(i)) Op.bind(r, target, t) } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy index d7c8462525..a4bed06c08 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy @@ -101,7 +101,7 @@ class Op { * is emitted in any case. */ static void bindRunValues(DataflowWriteChannel target, List entries, boolean singleton) { - final inputs = new ArrayList(entries.size()) + final inputs = new ArrayList(entries.size()) final values = new ArrayList(entries.size()) for( Object it : entries ) { if( it instanceof OpDatum ) { @@ -111,7 +111,7 @@ class Op { else values.add(it) } - final run = new OperatorRun(inputs) + final run = new OperatorRun(new LinkedHashSet(inputs)) final out = singleton && values.size()==1 ? values[0] : values Op.bind(run, target, out) } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/op/OpDatum.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/op/OpDatum.groovy index 677a89a08c..b022ba27db 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/op/OpDatum.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/op/OpDatum.groovy @@ -21,7 +21,9 @@ import groovy.transform.Canonical import nextflow.prov.OperatorRun /** - * + * Associated a data value acquired by an operator with the corresponding + * {@link OperatorRun} instance. + * * @author Paolo Di Tommaso */ @Canonical diff --git a/modules/nextflow/src/main/groovy/nextflow/prov/OperatorRun.groovy b/modules/nextflow/src/main/groovy/nextflow/prov/OperatorRun.groovy index fbbb52101e..e3016d05c9 100644 --- a/modules/nextflow/src/main/groovy/nextflow/prov/OperatorRun.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/prov/OperatorRun.groovy @@ -31,7 +31,7 @@ class OperatorRun implements TrailRun { /** * The list of (object) ids that was received as input by a operator run */ - List inputIds = new ArrayList<>(10) + Set inputIds = new LinkedHashSet<>(10) @Override String toString() { From 787349fd5b18151388eaee2ecc2fdbfce807201b Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sat, 18 Jan 2025 13:03:11 +0100 Subject: [PATCH 48/66] Add groupTuple operator Signed-off-by: Paolo Di Tommaso --- .../src/main/groovy/nextflow/Channel.groovy | 2 +- .../nextflow/extension/GroupTupleOp.groovy | 64 +++++++++++------- .../extension/GroupTupleOpTest.groovy | 66 +++++++++++++------ .../test/groovy/nextflow/prov/ProvTest.groovy | 40 +++++++++++ 4 files changed, 128 insertions(+), 44 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/Channel.groovy b/modules/nextflow/src/main/groovy/nextflow/Channel.groovy index ee1bb55f43..6cb22e436a 100644 --- a/modules/nextflow/src/main/groovy/nextflow/Channel.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/Channel.groovy @@ -522,7 +522,7 @@ class Channel { def groupChannel = isFlat ? new DataflowQueue<>() : CH.create() new GroupTupleOp(groupOpts, mapChannel) - .setTarget(groupChannel) + .withTarget(groupChannel) .apply() // -- flat the group resulting tuples diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/GroupTupleOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/GroupTupleOp.groovy index 7932618d36..d5bc0b7296 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/GroupTupleOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/GroupTupleOp.groovy @@ -16,10 +16,17 @@ package nextflow.extension + import groovy.util.logging.Slf4j import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowWriteChannel +import groovyx.gpars.dataflow.operator.DataflowProcessor import nextflow.Channel +import nextflow.extension.op.ContextRunPerThread +import nextflow.extension.op.Op +import nextflow.extension.op.OpContext +import nextflow.extension.op.OpDatum +import nextflow.prov.OperatorRun import nextflow.util.ArrayBag import nextflow.util.CacheHelper import nextflow.util.CheckHelper @@ -35,7 +42,6 @@ class GroupTupleOp { static private List GROUP_DEFAULT_INDEX = [0] - /** * Comparator used to sort tuple entries (when required) */ @@ -55,8 +61,9 @@ class GroupTupleOp { private sort - GroupTupleOp(Map params, DataflowReadChannel source) { + private OpContext context = new ContextRunPerThread() + GroupTupleOp(Map params, DataflowReadChannel source) { CheckHelper.checkParams('groupTuple', params, GROUP_TUPLE_PARAMS) channel = source @@ -68,7 +75,7 @@ class GroupTupleOp { defineComparator() } - GroupTupleOp setTarget(DataflowWriteChannel target) { + GroupTupleOp withTarget(DataflowWriteChannel target) { this.target = target return this } @@ -90,7 +97,7 @@ class GroupTupleOp { /* * Collects received values grouping by key */ - private void collect(List tuple) { + private void collectTuple(List tuple) { final key = tuple[indices] // the actual grouping key final len = tuple.size() @@ -102,6 +109,7 @@ class GroupTupleOp { return result } + final run = context.getOperatorRun() int count=-1 for( int i=0; i bindTuple(items, size ?: sizeBy(keys)) } - target.bind(Channel.STOP) + Op.bind(dp, target, Channel.STOP) } /* * bind collected items to the target channel */ private void bindTuple( List items, int sz ) { - - def tuple = new ArrayList(items) - + final tuple = new ArrayList(items) if( !remainder && sz>0 ) { // verify exist it contains 'size' elements - List list = items.find { it instanceof List } + def list = (List) items.find { it instanceof List } if( list.size() != sz ) { return } } - + // unwrap all "OpData" object and restore original values + final run = unwrapValues(tuple) + // sort the tuple content when a comparator is defined if( comparator ) { sortInnerLists(tuple, comparator) } + // finally bind the resulting tuple + Op.bind(run, target, tuple) + } + + static protected OperatorRun unwrapValues(List tuple) { + final inputs = new ArrayList() + + for( Object it : tuple ) { + if( it instanceof ArrayBag ) { + final bag = it + for( int i=0; i(inputs)) } /** * Define the comparator to be used depending the #sort property */ private void defineComparator( ) { - /* * comparator logic used to sort tuple elements */ @@ -210,7 +232,6 @@ class GroupTupleOp { default: throw new IllegalArgumentException("Not a valid sort argument: ${sort}") } - } /** @@ -228,7 +249,8 @@ class GroupTupleOp { */ new SubscribeOp() .withSource(channel) - .withOnNext(this.&collect) + .withContext(context) + .withOnNext(this.&collectTuple) .withOnComplete(this.&finalise) .apply() @@ -239,13 +261,11 @@ class GroupTupleOp { } private static sortInnerLists( List tuple, Comparator c ) { - for( int i=0; i @@ -33,7 +33,6 @@ class GroupTupleOpTest extends Specification { } def 'should reuse the same key' () { - given: def key1 = ['a', 'b', 'c'] def key2 = ['a', 'b', 'c'] @@ -63,7 +62,6 @@ class GroupTupleOpTest extends Specification { } def 'should reuse same key with GroupSize' () { - given: def key1 = [new GroupKey('A',1) ] def key2 = [new GroupKey('A',1) ] @@ -80,7 +78,6 @@ class GroupTupleOpTest extends Specification { map.get(key2) == 1 map.get(key3) == null map.getOrCreate(key2) { return 10 } == 1 - } def 'should fetch groupsize' () { @@ -100,6 +97,33 @@ class GroupTupleOpTest extends Specification { 0 | [] } + def 'should unwrap tuple values' () { + given: + def r1 = new OperatorRun(new HashSet([1,2])) + def r2 = new OperatorRun(new HashSet([2,3])) + def d1 = OpDatum.of('A', r1) + def d2 = OpDatum.of('B', r1) + def d3 = OpDatum.of('C', r2) + def d4 = OpDatum.of('D', r2) + and: + def bag1 = new ArrayBag(d1, d2) + def bag2 = new ArrayBag(d3, d4) + def key = Mock(GroupKey) + and: + def tuple = [key, bag1, bag2] + + when: + def result = GroupTupleOp.unwrapValues(tuple) + then: + tuple[0] == key + and: + tuple[1] == new ArrayBag<>('A','B') + and: + tuple[2] == new ArrayBag<>('C', 'D') + and: + result.inputIds == [1,2,3] as Set + + } def 'should group items using dyn group size' () { given: @@ -115,29 +139,29 @@ class GroupTupleOpTest extends Specification { // here the size is defined as operator argument def result = tuples.channel().groupTuple(size: 2) then: - result.val == [k1, ['a', 'b'] ] - result.val == [k1, ['d', 'c'] ] - result.val == [k2, ['x', 'y'] ] - result.val == Channel.STOP + result.unwrap() == [k1, ['a', 'b'] ] + result.unwrap() == [k1, ['d', 'c'] ] + result.unwrap() == [k2, ['x', 'y'] ] + result.unwrap() == Channel.STOP when: // here the size is inferred by the key itself result = tuples.channel().groupTuple() then: - result.val == [k1, ['a', 'b'] ] - result.val == [k1, ['d', 'c'] ] - result.val == [k2, ['x', 'y', 'z'] ] - result.val == Channel.STOP + result.unwrap() == [k1, ['a', 'b'] ] + result.unwrap() == [k1, ['d', 'c'] ] + result.unwrap() == [k2, ['x', 'y', 'z'] ] + result.unwrap() == Channel.STOP when: result = tuples.channel().groupTuple(remainder: true) then: - result.val == [k1, ['a', 'b'] ] - result.val == [k1, ['d', 'c'] ] - result.val == [k2, ['x', 'y', 'z'] ] - result.val == [k3, ['q']] - result.val == [k1, ['f']] - result.val == Channel.STOP + result.unwrap() == [k1, ['a', 'b'] ] + result.unwrap() == [k1, ['d', 'c'] ] + result.unwrap() == [k2, ['x', 'y', 'z'] ] + result.unwrap() == [k3, ['q']] + result.unwrap() == [k1, ['f']] + result.unwrap() == Channel.STOP } } diff --git a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy index 4c1b1f5ead..ec50f33513 100644 --- a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy @@ -950,4 +950,44 @@ class ProvTest extends Dsl2Spec { upstreamTasksOf('p2 (4)') .name == ['p1 (2)'] } + + def 'should track provenance with groupTuple operator' () { + when: + dsl_eval(globalConfig(), ''' + workflow { + channel.of( [1, 'A'], [1, 'B'], [2, 'C'], [3, 'B'], [1, 'C'], [2, 'A'], [3, 'D'] ) \ + | p1 \ + | groupTuple \ + | p2 + } + + // the groupTuple should emit the following values + // + // [1, [A, B, C]] + // [2, [C, A]] + // [3, [B, D]] + + process p1 { + input: val(x) + output: val(y) + exec: + y = x + } + + process p2 { + input: val(x) + exec: + println "$task.process ($task.index) = ${x}" + } + ''') + then: + upstreamTasksOf('p2 (1)') + .name == ['p1 (1)', 'p1 (2)', 'p1 (5)'] + and: + upstreamTasksOf('p2 (2)') + .name == ['p1 (3)', 'p1 (6)'] + and: + upstreamTasksOf('p2 (3)') + .name == ['p1 (4)', 'p1 (7)'] + } } From 08ac37f23754071b83769e1e1183653fb2ed6cb7 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sat, 18 Jan 2025 13:13:20 +0100 Subject: [PATCH 49/66] Add compile static to GroupTupleOp class [ci fast] Signed-off-by: Paolo Di Tommaso --- .../nextflow/extension/GroupTupleOp.groovy | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/GroupTupleOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/GroupTupleOp.groovy index d5bc0b7296..ce2218764c 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/GroupTupleOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/GroupTupleOp.groovy @@ -16,7 +16,7 @@ package nextflow.extension - +import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowWriteChannel @@ -36,18 +36,19 @@ import nextflow.util.CheckHelper * @author Paolo Di Tommaso */ @Slf4j +@CompileStatic class GroupTupleOp { - static private Map GROUP_TUPLE_PARAMS = [ by: [Integer, List], sort: [Boolean, 'true','natural','deep','hash',Closure,Comparator], size: Integer, remainder: Boolean ] + static final private Map GROUP_TUPLE_PARAMS = [ by: [Integer, List], sort: [Boolean, 'true','natural','deep','hash',Closure,Comparator], size: Integer, remainder: Boolean ] - static private List GROUP_DEFAULT_INDEX = [0] + static final private List GROUP_DEFAULT_INDEX = [0] /** * Comparator used to sort tuple entries (when required) */ private Comparator comparator - private int size + private Integer size private List indices @@ -68,7 +69,7 @@ class GroupTupleOp { channel = source indices = getGroupTupleIndices(params) - size = params?.size ?: 0 + size = params?.size as Integer ?: 0 remainder = params?.remainder ?: false sort = params?.sort @@ -162,7 +163,7 @@ class GroupTupleOp { } static protected OperatorRun unwrapValues(List tuple) { - final inputs = new ArrayList() + final inputs = new ArrayList() for( Object it : tuple ) { if( it instanceof ArrayBag ) { @@ -190,21 +191,21 @@ class GroupTupleOp { case true: case 'true': case 'natural': - comparator = { o1,o2 -> o1<=>o2 } as Comparator + comparator = { o1, o2 -> o1<=>o2 } as Comparator break; case 'hash': comparator = { o1, o2 -> - def h1 = CacheHelper.hasher(o1).hash() - def h2 = CacheHelper.hasher(o2).hash() + final h1 = CacheHelper.hasher(o1).hash() + final h2 = CacheHelper.hasher(o2).hash() return h1.asLong() <=> h2.asLong() } as Comparator break case 'deep': comparator = { o1, o2 -> - def h1 = CacheHelper.hasher(o1, CacheHelper.HashMode.DEEP).hash() - def h2 = CacheHelper.hasher(o2, CacheHelper.HashMode.DEEP).hash() + final h1 = CacheHelper.hasher(o1, CacheHelper.HashMode.DEEP).hash() + final h2 = CacheHelper.hasher(o2, CacheHelper.HashMode.DEEP).hash() return h1.asLong() <=> h2.asLong() } as Comparator break @@ -220,8 +221,8 @@ class GroupTupleOp { } else if( closure.getMaximumNumberOfParameters()==1 ) { comparator = { o1, o2 -> - def v1 = closure.call(o1) - def v2 = closure.call(o2) + final v1 = closure.call(o1) as Comparable + final v2 = closure.call(o2) as Comparable return v1 <=> v2 } as Comparator } From 61c153c01263443d08932562a41bae3699ae845b Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sun, 19 Jan 2025 21:26:22 +0100 Subject: [PATCH 50/66] Add bare minimal docs [ci skip] Signed-off-by: Paolo Di Tommaso --- .../groovy/nextflow/extension/op/ContextRunPerThread.groovy | 3 ++- .../groovy/nextflow/extension/op/ContextSequential.groovy | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/op/ContextRunPerThread.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/op/ContextRunPerThread.groovy index 602603fc42..ea23d14ea1 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/op/ContextRunPerThread.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/op/ContextRunPerThread.groovy @@ -22,7 +22,8 @@ import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import nextflow.prov.OperatorRun /** - * + * Implements an operator context that binds a new run to the current thread + * * @author Paolo Di Tommaso */ @Slf4j diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/op/ContextSequential.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/op/ContextSequential.groovy index 746773d62d..343bc1441a 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/op/ContextSequential.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/op/ContextSequential.groovy @@ -22,7 +22,9 @@ import java.util.concurrent.ConcurrentHashMap import nextflow.prov.OperatorRun /** - * + * Implements an operator context that expected a new run is allocated + * "sequentially" after the previous execution + * * @author Paolo Di Tommaso */ class ContextSequential implements OpContext { From fadab205a7f41280e106cc59a8db010d93d6a054 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Mon, 20 Jan 2025 23:44:18 +0100 Subject: [PATCH 51/66] Add until operator [ci fast] Signed-off-by: Paolo Di Tommaso --- .../nextflow/extension/OperatorImpl.groovy | 2 +- .../groovy/nextflow/extension/UntilOp.groovy | 30 +++++++++-------- .../test/groovy/nextflow/prov/ProvTest.groovy | 33 +++++++++++++++++++ 3 files changed, 50 insertions(+), 15 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy index 97fa34a405..facdd3315e 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy @@ -727,7 +727,7 @@ class OperatorImpl { * @return The tap resulting dataflow channel */ DataflowWriteChannel tap( final DataflowReadChannel source, final Closure holder ) { - def tap = new TapOp(source, holder).apply() + final tap = new TapOp(source, holder).apply() OpCall.current.get().outputs.addAll( tap.outputs ) return tap.result } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/UntilOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/UntilOp.groovy index 99a8670cf8..071131f635 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/UntilOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/UntilOp.groovy @@ -17,13 +17,12 @@ package nextflow.extension -import static nextflow.extension.DataflowHelper.* - import groovy.transform.CompileStatic import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowWriteChannel import groovyx.gpars.dataflow.operator.DataflowProcessor import nextflow.Channel +import nextflow.extension.op.Op import org.codehaus.groovy.runtime.typehandling.DefaultTypeTransformation /** * Implements Nextflow `until` operator @@ -43,19 +42,22 @@ class UntilOp { DataflowWriteChannel apply() { final target = CH.createBy(source) - - newOperator(source, target, { - final result = DefaultTypeTransformation.castToBoolean(closure.call(it)) - final proc = getDelegate() as DataflowProcessor - - if( result ) { - proc.bindOutput(Channel.STOP) - proc.terminate() - } - else { - proc.bindOutput(it) + + new Op() + .withInput(source) + .withOutput(target) + .withCode { + final result = DefaultTypeTransformation.castToBoolean(closure.call(it)) + final dp = getDelegate() as DataflowProcessor + if( result ) { + Op.bind(dp, target, Channel.STOP) + dp.terminate() + } + else { + Op.bind(dp, target, it) + } } - }) + .apply() return target } diff --git a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy index ec50f33513..5f32850e16 100644 --- a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy @@ -990,4 +990,37 @@ class ProvTest extends Dsl2Spec { upstreamTasksOf('p2 (3)') .name == ['p1 (4)', 'p1 (7)'] } + + def 'should track provenance with until operator'() { + + when: + dsl_eval(globalConfig(), ''' + workflow { + channel.of(1,2,3,4) | p1 | until{ it->it>3 } | p2 + } + + process p1 { + input: val(x) + output: val(y) + exec: + y = x + } + + process p2 { + input: val(x) + exec: + println x + } + ''') + + then: + upstreamTasksOf('p2 (1)') + .name == ['p1 (1)'] + and: + upstreamTasksOf('p2 (2)') + .name == ['p1 (2)'] + and: + upstreamTasksOf('p2 (3)') + .name == ['p1 (3)'] + } } From 0e30a8f0b50ba3e800631b6708f100cf2c7b9f33 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 21 Jan 2025 00:02:42 +0100 Subject: [PATCH 52/66] Add ifEmpty operator [ci fast] Signed-off-by: Paolo Di Tommaso --- .../nextflow/extension/OperatorImpl.groovy | 35 ++++++++++--------- .../test/groovy/nextflow/prov/ProvTest.groovy | 33 ++++++++++++++++- 2 files changed, 50 insertions(+), 18 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy index facdd3315e..86d074bf67 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy @@ -28,10 +28,13 @@ import groovyx.gpars.dataflow.DataflowVariable import groovyx.gpars.dataflow.DataflowWriteChannel import groovyx.gpars.dataflow.expression.DataflowExpression import groovyx.gpars.dataflow.operator.ChainWithClosure +import groovyx.gpars.dataflow.operator.DataflowProcessor import nextflow.Channel import nextflow.Global import nextflow.NF import nextflow.Session +import nextflow.extension.op.ContextRunPerThread +import nextflow.extension.op.Op import nextflow.script.ChannelOut import nextflow.script.TokenBranchDef import nextflow.script.TokenMultiMapDef @@ -587,9 +590,9 @@ class OperatorImpl { throw new IllegalArgumentException("Illegal argument 'size' for operator 'collate' -- it must be greater than zero: $size") } - return new BufferOp(source) - .setParams( size: size, remainder: keepRemainder ) - .apply() + new BufferOp(source) + .setParams( size: size, remainder: keepRemainder ) + .apply() } DataflowWriteChannel collate( DataflowReadChannel source, int size, int step, boolean keepRemainder = true ) { @@ -622,8 +625,7 @@ class OperatorImpl { // therefore the channel need to be added `'manually` to the inputs list // fixes #1346 OpCall.current.get().inputs.add(right) - def target = new JoinOp(left,right) .apply() - return target + return new JoinOp(left,right) .apply() } DataflowWriteChannel join( DataflowReadChannel left, Map opts, right ) { @@ -633,8 +635,7 @@ class OperatorImpl { // therefore the channel need to be added `'manually` to the inputs list // fixes #1346 OpCall.current.get().inputs.add(right) - def target = new JoinOp(left,right,opts) .apply() - return target + return new JoinOp(left,right,opts) .apply() } /** @@ -732,7 +733,6 @@ class OperatorImpl { return tap.result } - /** * Empty the specified value only if the source channel to which is applied is empty i.e. do not emit * any value. @@ -746,18 +746,19 @@ class OperatorImpl { boolean empty = true final result = CH.createBy(source) final singleton = result instanceof DataflowExpression - final next = { result.bind(it); empty=false } - final complete = { - if(empty) - result.bind( value instanceof Closure ? value() : value ) - if( !singleton ) - result.bind(Channel.STOP) - } new SubscribeOp() .withSource(source) - .withOnNext(next) - .withOnComplete(complete) + .withContext(new ContextRunPerThread()) + .withOnNext { DataflowProcessor dp, Object it -> Op.bind(dp,result,it); empty=false } + .withOnComplete { DataflowProcessor dp -> + if(empty) { + final x = value instanceof Closure ? value.call() : value + Op.bind(dp,result,x) + } + if( !singleton ) + result.bind(Channel.STOP) + } .apply() return result diff --git a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy index 5f32850e16..125f07700e 100644 --- a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy @@ -992,7 +992,6 @@ class ProvTest extends Dsl2Spec { } def 'should track provenance with until operator'() { - when: dsl_eval(globalConfig(), ''' workflow { @@ -1023,4 +1022,36 @@ class ProvTest extends Dsl2Spec { upstreamTasksOf('p2 (3)') .name == ['p1 (3)'] } + + def 'should track provenance with ifEmpty operator'() { + when: + dsl_eval(globalConfig(), ''' + workflow { + channel.of(1,2,3) | p1 | ifEmpty('nope') | p2 + } + + process p1 { + input: val(x) + output: val(y) + exec: + y = x + } + + process p2 { + input: val(x) + exec: + println x + } + ''') + + then: + upstreamTasksOf('p2 (1)') + .name == ['p1 (1)'] + and: + upstreamTasksOf('p2 (2)') + .name == ['p1 (2)'] + and: + upstreamTasksOf('p2 (3)') + .name == ['p1 (3)'] + } } From 213e1dae25eef5357f474772f656af8cd01e85b9 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sat, 1 Feb 2025 08:51:19 +0100 Subject: [PATCH 53/66] split wip Signed-off-by: Paolo Di Tommaso --- .../groovy/nextflow/extension/SplitOp.groovy | 14 ++-- .../extension/SplitterMergeClosure.groovy | 28 +++++--- .../nextflow/splitter/AbstractSplitter.groovy | 19 +++-- .../nextflow/splitter/FastqSplitter.groovy | 2 +- .../extension/SplitFastaOperatorTest.groovy | 33 +++------ .../extension/SplitFastqOp2Test.groovy | 13 ++-- .../nextflow/extension/SplitTextOpTest.groovy | 32 +++++---- .../extension/SplitterMergeClosureTest.groovy | 7 +- .../test/groovy/nextflow/prov/ProvTest.groovy | 71 ++++++++++++++++++- 9 files changed, 150 insertions(+), 69 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/SplitOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/SplitOp.groovy index 4c6b99e685..c3360ae0da 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/SplitOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/SplitOp.groovy @@ -22,7 +22,9 @@ import groovy.util.logging.Slf4j import groovyx.gpars.dataflow.DataflowQueue import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowWriteChannel +import groovyx.gpars.dataflow.operator.DataflowProcessor import nextflow.Channel +import nextflow.extension.op.Op import nextflow.splitter.AbstractSplitter import nextflow.splitter.FastqSplitter import nextflow.splitter.SplitterFactory @@ -164,7 +166,7 @@ class SplitOp { params.into = output // -- create the splitter and set the options - def splitter = createSplitter(methodName, params) + final splitter = createSplitter(methodName, params) // -- specify if it's a multi-file splitting operation if( multiSplit ) @@ -188,8 +190,8 @@ class SplitOp { void applySplittingOperator( DataflowReadChannel origin, DataflowWriteChannel output, AbstractSplitter splitter ) { new SubscribeOp() .withSource(origin) - .withOnNext({ entry -> splitter.target(entry).apply() }) - .withOnComplete({ output << Channel.STOP }) + .withOnNext({ DataflowProcessor dp, entry -> splitter.processor(dp).target(entry).apply() }) + .withOnComplete({ DataflowProcessor dp -> Op.bind(dp,output,Channel.STOP) }) .apply() } @@ -202,7 +204,11 @@ class SplitOp { @PackageScope void applyMergingOperator(List splitted, DataflowWriteChannel output, List indexes) { - DataflowHelper.newOperator(splitted, [output], new SplitterMergeClosure(indexes)) + new Op() + .withInputs(splitted) + .withOutput(output) + .withCode(new SplitterMergeClosure(indexes, output)) + .apply() } @PackageScope diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/SplitterMergeClosure.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/SplitterMergeClosure.groovy index 7ee4be8f9f..909738647e 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/SplitterMergeClosure.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/SplitterMergeClosure.groovy @@ -16,8 +16,11 @@ package nextflow.extension + import groovy.util.logging.Slf4j +import groovyx.gpars.dataflow.DataflowWriteChannel import groovyx.gpars.dataflow.operator.DataflowProcessor +import nextflow.extension.op.Op import nextflow.splitter.FastqSplitter /** * Implements the inner merging logic for splitterXxx operator(s) @@ -33,39 +36,42 @@ class SplitterMergeClosure extends Closure { private int emissionCount - SplitterMergeClosure(List indexes) { + private DataflowWriteChannel target + + SplitterMergeClosure(List indexes, DataflowWriteChannel target) { super(null, null); this.numOfParams = indexes.size() this.indexes = indexes + this.target = target } @Override - public int getMaximumNumberOfParameters() { + int getMaximumNumberOfParameters() { numOfParams } @Override - public Class[] getParameterTypes() { + Class[] getParameterTypes() { Collections.nCopies(numOfParams, Object) } @Override - public void setDelegate(final Object delegate) { + void setDelegate(final Object delegate) { super.setDelegate(delegate); } @Override - public void setResolveStrategy(final int resolveStrategy) { + void setResolveStrategy(final int resolveStrategy) { super.setResolveStrategy(resolveStrategy); } @Override - public Object call(final Object arguments) { + Object call(final Object arguments) { throw new UnsupportedOperationException() } @Override - public Object call(final Object... args) { + Object call(final Object... args) { log.trace "merging ($emissionCount) indexes=$indexes; args=$args" List result = null boolean header = false @@ -93,10 +99,10 @@ class SplitterMergeClosure extends Closure { } // emit the merged tuple (skipping the header) - def processor = (DataflowProcessor)getDelegate() - if( !header && processor ) { + final dp = getDelegate() as DataflowProcessor + if( !header && dp ) { log.trace "merging ($emissionCount) result=$result" - processor.bindAllOutputsAtomically(result); + Op.bind(dp, target, result) } emissionCount++ @@ -105,7 +111,7 @@ class SplitterMergeClosure extends Closure { } @Override - public Object call() { + Object call() { throw new UnsupportedOperationException() } } diff --git a/modules/nextflow/src/main/groovy/nextflow/splitter/AbstractSplitter.groovy b/modules/nextflow/src/main/groovy/nextflow/splitter/AbstractSplitter.groovy index 09320afa59..8d1263516e 100644 --- a/modules/nextflow/src/main/groovy/nextflow/splitter/AbstractSplitter.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/splitter/AbstractSplitter.groovy @@ -26,9 +26,11 @@ import groovy.util.logging.Slf4j import groovyx.gpars.dataflow.DataflowBroadcast import groovyx.gpars.dataflow.DataflowQueue import groovyx.gpars.dataflow.DataflowWriteChannel +import groovyx.gpars.dataflow.operator.DataflowProcessor import nextflow.Channel import nextflow.exception.StopSplitIterationException import nextflow.extension.CH +import nextflow.extension.op.Op import nextflow.util.CheckHelper /** * Generic data splitter, provide main methods/interfaces @@ -41,7 +43,7 @@ abstract class AbstractSplitter implements SplitterStrategy { protected Map fOptionsMap - protected def into + protected Object into protected Closure closure @@ -53,7 +55,7 @@ abstract class AbstractSplitter implements SplitterStrategy { protected Path sourceFile - protected decompress + protected Object decompress protected String operatorName @@ -61,7 +63,9 @@ abstract class AbstractSplitter implements SplitterStrategy { protected Integer elem - private targetObj + private Object targetObj + + private DataflowProcessor processor private CollectorStrategy collector @@ -332,6 +336,11 @@ abstract class AbstractSplitter implements SplitterStrategy { return this } + AbstractSplitter processor(DataflowProcessor processor) { + this.processor = processor + return this + } + /** * Start the slitting */ @@ -424,7 +433,7 @@ abstract class AbstractSplitter implements SplitterStrategy { into.add(value) else if( into instanceof DataflowWriteChannel ) - into.bind(value) + Op.bind(processor, into, value) else throw new IllegalArgumentException("Not a valid 'into' target object: ${into?.class?.name}") @@ -467,7 +476,7 @@ abstract class AbstractSplitter implements SplitterStrategy { * @return The current {@link CollectorStrategy} object */ final protected CollectorStrategy getCollector() { - collector + return collector } /** diff --git a/modules/nextflow/src/main/groovy/nextflow/splitter/FastqSplitter.groovy b/modules/nextflow/src/main/groovy/nextflow/splitter/FastqSplitter.groovy index 8d8e43d50f..7cdd9e5c80 100644 --- a/modules/nextflow/src/main/groovy/nextflow/splitter/FastqSplitter.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/splitter/FastqSplitter.groovy @@ -101,7 +101,7 @@ class FastqSplitter extends AbstractTextSplitter { } - def private StringBuilder buffer = new StringBuilder() + private StringBuilder buffer = new StringBuilder() private String errorMessage = "Invalid FASTQ format" diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/SplitFastaOperatorTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/SplitFastaOperatorTest.groovy index 65f0f38c6e..c1bdcc77f9 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/SplitFastaOperatorTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/SplitFastaOperatorTest.groovy @@ -15,6 +15,8 @@ */ package nextflow.extension + +import groovyx.gpars.dataflow.DataflowReadChannel import nextflow.Channel import nextflow.Session import spock.lang.Shared @@ -67,23 +69,17 @@ class SplitFastaOperatorTest extends Specification { """.stripIndent() def 'should split fasta in sequences'() { - given: - def sequences = Channel.of(fasta1).splitFasta() + def sequences = Channel.of(fasta1).splitFasta() as DataflowReadChannel expect: - with(sequences) { - val == '>1aboA\nNLFVALYDFVASGDNTLSITKGEKLRVLGYNHNGEWCEAQTKNGQGWVPS\nNYITPVN\n' - val == '>1ycsB\nKGVIYALWDYEPQNDDELPMKEGDCMTIIHREDEDEIEWWWARLNDKEGY\nVPRNLLGLYP\n' - val == '>1pht\nGYQYRALYDYKKEREEDIDLHLGDILTVNKGSLVALGFSDGQEARPEEIG\nWLNGYNETTGERGDFPGTYVEYIGRKKISP\n' - val == Channel.STOP - } - + sequences.unwrap() == '>1aboA\nNLFVALYDFVASGDNTLSITKGEKLRVLGYNHNGEWCEAQTKNGQGWVPS\nNYITPVN\n' + sequences.unwrap() == '>1ycsB\nKGVIYALWDYEPQNDDELPMKEGDCMTIIHREDEDEIEWWWARLNDKEGY\nVPRNLLGLYP\n' + sequences.unwrap() == '>1pht\nGYQYRALYDYKKEREEDIDLHLGDILTVNKGSLVALGFSDGQEARPEEIG\nWLNGYNETTGERGDFPGTYVEYIGRKKISP\n' + sequences.unwrap() == Channel.STOP } - def 'should split fasta in records' () { - given: def records = Channel.of(fasta1, fasta2).splitFasta(record:[id:true]) expect: @@ -92,14 +88,12 @@ class SplitFastaOperatorTest extends Specification { records.unwrap() == [id:'1pht'] records.unwrap() == [id:'alpha123'] records.unwrap() == Channel.STOP - } def 'should split tuple in fasta records' () { - given: def result = Channel - .from( [fasta1, 'one'], [fasta2,'two'] ) + .of( [fasta1, 'one'], [fasta2,'two'] ) .splitFasta(record:[id:true]) .map{ record, code -> [record.id, code] } @@ -109,13 +103,11 @@ class SplitFastaOperatorTest extends Specification { result.unwrap() == ['1pht', 'one'] result.unwrap() == ['alpha123', 'two'] result.unwrap() == Channel.STOP - } def 'should split fasta and forward result into the specified channel' () { - given: - def target = Channel.create() + def target = CH.queue() Channel.of(fasta1,fasta2).splitFasta(record:[id:true], into: target) expect: @@ -126,9 +118,7 @@ class SplitFastaOperatorTest extends Specification { target.unwrap() == Channel.STOP } - def 'should apply count on multiple entries'() { - given: def F1 = ''' >1 @@ -149,7 +139,7 @@ class SplitFastaOperatorTest extends Specification { ''' .stripIndent().trim() - def target = Channel.create() + def target = CH.queue() when: Channel.of(F1,F3).splitFasta(by:2, into: target) @@ -161,7 +151,6 @@ class SplitFastaOperatorTest extends Specification { } def 'should apply count on multiple entries with a limit'() { - given: def F1 = ''' >1 @@ -190,7 +179,7 @@ class SplitFastaOperatorTest extends Specification { ''' .stripIndent().trim() - def target = Channel.create() + def target = CH.queue() when: Channel.of(F1,F3).splitFasta(by:2, limit:4, into: target) diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/SplitFastqOp2Test.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/SplitFastqOp2Test.groovy index 3391b222f1..9a818e8c86 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/SplitFastqOp2Test.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/SplitFastqOp2Test.groovy @@ -18,6 +18,7 @@ package nextflow.extension import java.nio.file.Files +import groovyx.gpars.dataflow.DataflowReadChannel import nextflow.Channel import test.Dsl2Spec /** @@ -77,9 +78,9 @@ class SplitFastqOp2Test extends Dsl2Spec { when: channel = dsl_eval(""" Channel.of(['sample_id', file("$file1"), file("$file2")]).splitFastq(by:1, pe:true) - """) + """) as DataflowReadChannel - result = channel.val + result = channel.unwrap() then: result[0] == 'sample_id' @@ -97,7 +98,7 @@ class SplitFastqOp2Test extends Dsl2Spec { '''.stripIndent().leftTrim() when: - result = channel.val + result = channel.unwrap() then: result[0] == 'sample_id' result[1] == ''' @@ -114,7 +115,7 @@ class SplitFastqOp2Test extends Dsl2Spec { '''.stripIndent().leftTrim() when: - result = channel.val + result = channel.unwrap() then: result[0] == 'sample_id' result[1] == ''' @@ -131,7 +132,7 @@ class SplitFastqOp2Test extends Dsl2Spec { '''.stripIndent().leftTrim() when: - result = channel.val + result = channel.unwrap() then: result[0] == 'sample_id' result[1] == ''' @@ -148,7 +149,7 @@ class SplitFastqOp2Test extends Dsl2Spec { '''.stripIndent().leftTrim() when: - result = channel.val + result = channel.unwrap() then: result == Channel.STOP diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/SplitTextOpTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/SplitTextOpTest.groovy index 7c99c173aa..58f1268262 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/SplitTextOpTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/SplitTextOpTest.groovy @@ -1,5 +1,6 @@ package nextflow.extension +import nextflow.Session import spock.lang.Specification import nextflow.Channel @@ -10,39 +11,40 @@ import nextflow.Channel */ class SplitTextOpTest extends Specification { - def 'should split text' () { + def setupSpec() { + new Session() + } + def 'should split text' () { when: def result = Channel.of('foo\nbar').splitText() then: - result.val == 'foo\n' - result.val == 'bar\n' - result.val == Channel.STOP + result.unwrap() == 'foo\n' + result.unwrap() == 'bar\n' + result.unwrap() == Channel.STOP when: result = Channel.of('foo\nbar\nbaz').splitText(by:2) then: - result.val == 'foo\nbar\n' - result.val == 'baz\n' - result.val == Channel.STOP + result.unwrap() == 'foo\nbar\n' + result.unwrap() == 'baz\n' + result.unwrap() == Channel.STOP } def 'should split text and invoke closure' () { - when: def result = Channel.of('foo\nbar').splitText { it.trim().reverse() } then: - result.val == 'oof' - result.val == 'rab' - result.val == Channel.STOP + result.unwrap() == 'oof' + result.unwrap() == 'rab' + result.unwrap() == Channel.STOP when: result = Channel.of('aa\nbb\ncc\ndd').splitText(by:2) { it.trim() } then: - result.val == 'aa\nbb' - result.val == 'cc\ndd' - result.val == Channel.STOP + result.unwrap() == 'aa\nbb' + result.unwrap() == 'cc\ndd' + result.unwrap() == Channel.STOP } - } diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/SplitterMergeClosureTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/SplitterMergeClosureTest.groovy index d6b9351d6b..29f7c33793 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/SplitterMergeClosureTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/SplitterMergeClosureTest.groovy @@ -16,6 +16,7 @@ package nextflow.extension +import groovyx.gpars.dataflow.DataflowWriteChannel import spock.lang.Specification /** * @@ -26,7 +27,7 @@ class SplitterMergeClosureTest extends Specification { def 'should merge splits' () { when: - def merge = new SplitterMergeClosure([1,2]) + def merge = new SplitterMergeClosure([1,2], Mock(DataflowWriteChannel)) def output1 = ['pair_1', 'a', ['x','y'], 'any'] def output2 = ['pair_1', ['a','b'], 'x', 'any'] def result = merge.call( [output1, output2] as Object[] ) @@ -34,7 +35,7 @@ class SplitterMergeClosureTest extends Specification { result == ['pair_1', 'a', 'x', 'any'] when: - merge = new SplitterMergeClosure([2,3]) + merge = new SplitterMergeClosure([2,3], Mock(DataflowWriteChannel)) output1 = ['pair_1', 'any' , 'a', ['x','y']] output2 = ['pair_1', 'any', ['a','b'], 'x'] result = merge.call( [output1, output2] as Object[] ) @@ -42,7 +43,7 @@ class SplitterMergeClosureTest extends Specification { result == ['pair_1', 'any', 'a', 'x'] when: - merge = new SplitterMergeClosure([0,1]) + merge = new SplitterMergeClosure([0,1], Mock(DataflowWriteChannel)) output1 = ['a', ['x','y'], 'pair_1', 'any' ] output2 = [['a','b'], 'x', 'pair_1', 'any'] result = merge.call( [output1, output2] as Object[] ) diff --git a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy index 125f07700e..f78e71b8ba 100644 --- a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy @@ -535,7 +535,7 @@ class ProvTest extends Dsl2Spec { process p2 { input: val(x) exec: - println "$task.process ($task.index) = ${x}" + println "${task.name} = ${x}" } ''') @@ -977,7 +977,7 @@ class ProvTest extends Dsl2Spec { process p2 { input: val(x) exec: - println "$task.process ($task.index) = ${x}" + println "${task.name} = ${x}" } ''') then: @@ -1054,4 +1054,71 @@ class ProvTest extends Dsl2Spec { upstreamTasksOf('p2 (3)') .name == ['p1 (3)'] } + + def 'should track provenance with splitText operator'() { + when: + dsl_eval(globalConfig(), ''' + workflow { + Channel.of('aa\\nbb\\ncc') | p1 | splitText | p2 + } + + process p1 { + input: val(x) + output: val(y) + exec: + y = x + } + + process p2 { + input: val(x) + exec: + println x + } + ''') + + then: + upstreamTasksOf('p2 (1)') + .name == ['p1 (1)'] + and: + upstreamTasksOf('p2 (2)') + .name == ['p1 (1)'] + and: + upstreamTasksOf('p2 (3)') + .name == ['p1 (1)'] + } + + def 'should track provenance with splitText and a file'() { + given: + + when: + dsl_eval(globalConfig(), """ + workflow { + p1 | splitText | p2 + } + + process p1 { + output: path('result.txt') + exec: + task.workDir.resolve('result.txt').text = 'a\\nbb\\nccc\\n' + } + + process p2 { + input: val(chunk) + exec: + println "\${task.name}: chunck=\$chunk" + } + """) + + then: + upstreamTasksOf('p2 (1)') + .name == ['p1'] + and: + upstreamTasksOf('p2 (2)') + .name == ['p1'] + and: + upstreamTasksOf('p2 (3)') + .name == ['p1'] + + } + } From d8625d17772ecb0ffcd4200e1d91ee29cc75b51b Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sun, 2 Feb 2025 14:00:17 +0100 Subject: [PATCH 54/66] wip Signed-off-by: Paolo Di Tommaso --- .../extension/SplitFastqOp2Test.groovy | 6 +-- .../test/groovy/nextflow/prov/ProvTest.groovy | 47 +++++++++++++++++++ 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/SplitFastqOp2Test.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/SplitFastqOp2Test.groovy index 9a818e8c86..56f9e1e67e 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/SplitFastqOp2Test.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/SplitFastqOp2Test.groovy @@ -27,7 +27,7 @@ import test.Dsl2Spec */ class SplitFastqOp2Test extends Dsl2Spec { - String READS = ''' + static String READS1 = ''' @SRR636272.19519409/1 GGCCCGGCAGCAGGATGATGCTCTCCCGGGCCAAGCCGGCTGTGGGGAGCACCCCGCCGCAGGGGGACAGGCGGAGGAAGAAAGGGAAGAAGGTGCCACAGATCG + @@ -46,7 +46,7 @@ class SplitFastqOp2Test extends Dsl2Spec { CCCFFFFFHHHHHJJJJJJJJJJJJJJJHFDDBDDBDDDDDDDDDDDDADDDDDDDDDDDDDDDDDDDDDDDDDDBDBDDD9@DDDDDDDDDDDDBBDDDBDD@@ '''.stripIndent().leftTrim() - String READS2 = ''' + static String READS2 = ''' @SRR636272.19519409/2 GGCCCGGCAGCAGGATGATGCTCTCCCGGGCCAAGCCGGCTGTGGGGAGCACCCCGCCGCAGGGGGACAGGCGGAGGAAGAAAGGGAAGAAGGTGCCACAGATCG + @@ -69,7 +69,7 @@ class SplitFastqOp2Test extends Dsl2Spec { def 'should split pair-ended using dsl2' () { given: def folder = Files.createTempDirectory('test') - def file1 = folder.resolve('one.fq'); file1.text = READS + def file1 = folder.resolve('one.fq'); file1.text = READS1 def file2 = folder.resolve('two.fq'); file2.text = READS2 def result diff --git a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy index f78e71b8ba..d2781a85a7 100644 --- a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy @@ -3,6 +3,7 @@ package nextflow.prov import static test.TestHelper.* import nextflow.config.ConfigParser +import nextflow.extension.SplitFastqOp2Test import nextflow.processor.TaskId import nextflow.processor.TaskProcessor import spock.lang.Ignore @@ -1121,4 +1122,50 @@ class ProvTest extends Dsl2Spec { } + def 'should track provenance with splitFastq paired'() { + given: + + when: + dsl_eval(globalConfig(), """ + workflow { + p1() + p2() + p1.out | join(p2.out) | splitFastq(by:1, pe:true, file:true) | p3 + } + + process p1 { + output: tuple val(sample), path('read1.fq') + exec: + sample = 'sample_1' + task.workDir.resolve('read1.fq').text = '''${SplitFastqOp2Test.READS1}''' + } + + process p2 { + output: tuple val(sample), path('read2.fq') + exec: + sample = 'sample_1' + task.workDir.resolve('read2.fq').text = '''${SplitFastqOp2Test.READS2}''' + } + + process p3 { + input: tuple val(sample), path(r1), path(r2) + exec: + println "\${task.name}: sample=\${sample};" + } + + """) + + then: + noExceptionThrown() + upstreamTasksOf('p3 (1)') + .name == ['p1 (1)', 'p2 (1)'] +// and: +// upstreamTasksOf('p2 (2)') +// .name == ['p1'] +// and: +// upstreamTasksOf('p2 (3)') +// .name == ['p1'] + + } + } From 8ff5caa1b742f20bf68e3b22992d41c5b285fdd1 Mon Sep 17 00:00:00 2001 From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Date: Sat, 8 Feb 2025 14:35:21 -0500 Subject: [PATCH 55/66] Allow Azure Batch tasks to be submitted to different pools (#5766) [ci fast] Signed-off-by: adamrtalbot <12817534+adamrtalbot@users.noreply.github.com> Signed-off-by: Paolo Di Tommaso Co-authored-by: Paolo Di Tommaso --- .../cloud/azure/batch/AzBatchService.groovy | 20 +++++--- .../cloud/azure/batch/AzJobKey.groovy | 33 ++++++++++++ .../azure/batch/AzBatchServiceTest.groovy | 50 +++++++++++++++++++ .../cloud/azure/batch/AzJobKeyTest.groovy | 48 ++++++++++++++++++ 4 files changed, 145 insertions(+), 6 deletions(-) create mode 100644 plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzJobKey.groovy create mode 100644 plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzJobKeyTest.groovy diff --git a/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzBatchService.groovy b/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzBatchService.groovy index 26a87a8177..2a54093d68 100644 --- a/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzBatchService.groovy +++ b/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzBatchService.groovy @@ -88,7 +88,6 @@ import nextflow.cloud.types.CloudMachineInfo import nextflow.cloud.types.PriceModel import nextflow.fusion.FusionHelper import nextflow.fusion.FusionScriptLauncher -import nextflow.processor.TaskProcessor import nextflow.processor.TaskRun import nextflow.util.CacheHelper import nextflow.util.MemoryUnit @@ -111,7 +110,7 @@ class AzBatchService implements Closeable { AzConfig config - Map allJobIds = new HashMap<>(50) + Map allJobIds = new HashMap<>(50) AzBatchService(AzBatchExecutor executor) { assert executor @@ -355,17 +354,26 @@ class AzBatchService implements Closeable { } synchronized String getOrCreateJob(String poolId, TaskRun task) { - final mapKey = task.processor + // Use the same job Id for the same Process,PoolId pair + // The Pool is added to allow using different queue names (corresponding + // a pool id) for the same process. See also + // https://github.com/nextflow-io/nextflow/pull/5766 + final mapKey = new AzJobKey(task.processor, poolId) if( allJobIds.containsKey(mapKey)) { return allJobIds[mapKey] } + final jobId = createJob0(poolId,task) + // add to the map + allJobIds[mapKey] = jobId + return jobId + } + + protected String createJob0(String poolId, TaskRun task) { + log.debug "[AZURE BATCH] created job for ${task.processor.name} with pool ${poolId}" // create a batch job final jobId = makeJobId(task) final content = new BatchJobCreateContent(jobId, new BatchPoolInfo(poolId: poolId)) apply(() -> client.createJob(content)) - // add to the map - allJobIds[mapKey] = jobId - return jobId } String makeJobId(TaskRun task) { diff --git a/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzJobKey.groovy b/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzJobKey.groovy new file mode 100644 index 0000000000..3957ca7f55 --- /dev/null +++ b/plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzJobKey.groovy @@ -0,0 +1,33 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.cloud.azure.batch + +import groovy.transform.Canonical +import groovy.transform.CompileStatic +import nextflow.processor.TaskProcessor +/** + * Model a Batch job key for caching purposes + * + * @author Paolo Di Tommaso + */ +@Canonical +@CompileStatic +class AzJobKey { + final TaskProcessor processor + final String poolId +} diff --git a/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzBatchServiceTest.groovy b/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzBatchServiceTest.groovy index eceeb644b2..e5431abf12 100644 --- a/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzBatchServiceTest.groovy +++ b/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzBatchServiceTest.groovy @@ -739,4 +739,54 @@ class AzBatchServiceTest extends Specification { [managedIdentity: [clientId: 'client-123']] | 'client-123' } + def 'should cache job id' () { + given: + def exec = Mock(AzBatchExecutor) + def service = Spy(new AzBatchService(exec)) + and: + def p1 = Mock(TaskProcessor) + def p2 = Mock(TaskProcessor) + def t1 = Mock(TaskRun) { getProcessor()>>p1 } + def t2 = Mock(TaskRun) { getProcessor()>>p2 } + def t3 = Mock(TaskRun) { getProcessor()>>p2 } + + when: + def result = service.getOrCreateJob('foo',t1) + then: + 1 * service.createJob0('foo',t1) >> 'job1' + and: + result == 'job1' + + // second time is cached + when: + result = service.getOrCreateJob('foo',t1) + then: + 0 * service.createJob0('foo',t1) >> null + and: + result == 'job1' + + // changing pool id returns a new job id + when: + result = service.getOrCreateJob('bar',t1) + then: + 1 * service.createJob0('bar',t1) >> 'job2' + and: + result == 'job2' + + // changing process returns a new job id + when: + result = service.getOrCreateJob('bar',t2) + then: + 1 * service.createJob0('bar',t2) >> 'job3' + and: + result == 'job3' + + // change task with the same process, return cached job id + when: + result = service.getOrCreateJob('bar',t3) + then: + 0 * service.createJob0('bar',t3) >> null + and: + result == 'job3' + } } diff --git a/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzJobKeyTest.groovy b/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzJobKeyTest.groovy new file mode 100644 index 0000000000..225230ee8b --- /dev/null +++ b/plugins/nf-azure/src/test/nextflow/cloud/azure/batch/AzJobKeyTest.groovy @@ -0,0 +1,48 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.cloud.azure.batch + +import nextflow.processor.TaskProcessor +import spock.lang.Specification + +/** + * + * @author Paolo Di Tommaso + */ +class AzJobKeyTest extends Specification { + + def 'should validate equals and hashcode' () { + given: + def p1 = Mock(TaskProcessor) + def p2 = Mock(TaskProcessor) + def k1 = new AzJobKey(p1, 'foo') + def k2 = new AzJobKey(p1, 'foo') + def k3 = new AzJobKey(p2, 'foo') + def k4 = new AzJobKey(p1, 'bar') + + expect: + k1 == k2 + k1 != k3 + k1 != k4 + and: + k1.hashCode() == k2.hashCode() + k1.hashCode() != k3.hashCode() + k1.hashCode() != k4.hashCode() + } + +} From a4687c443adb4fa99f73af49ae6518c6c673b1ce Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sat, 8 Feb 2025 20:38:48 +0100 Subject: [PATCH 56/66] Disable cleanup on remote work dir (#5742) Signed-off-by: Paolo Di Tommaso Signed-off-by: Ben Sherman Co-authored-by: Ben Sherman Co-authored-by: Chris Hakkaart --- docs/reference/config.md | 5 ++++- modules/nextflow/src/main/groovy/nextflow/Session.groovy | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/reference/config.md b/docs/reference/config.md index 25a0bdb797..e653e3078d 100644 --- a/docs/reference/config.md +++ b/docs/reference/config.md @@ -12,7 +12,10 @@ This page lists all of the available settings in the {ref}`Nextflow configuratio : If `true`, on a successful completion of a run all files in *work* directory are automatically deleted. :::{warning} - The use of the `cleanup` option will prevent the use of the *resume* feature on subsequent executions of that pipeline run. Also, be aware that deleting all scratch files can take a lot of time, especially when using a shared file system or remote cloud storage. + The use of the `cleanup` option will prevent the use of the *resume* feature on subsequent executions of that pipeline run. + ::: + :::{warning} + The `cleanup` option is not supported for remote work directories, such as Amazon S3, Google Cloud Storage, and Azure Blob Storage. ::: `dumpHashes` diff --git a/modules/nextflow/src/main/groovy/nextflow/Session.groovy b/modules/nextflow/src/main/groovy/nextflow/Session.groovy index 5b2eee8e71..09152bc301 100644 --- a/modules/nextflow/src/main/groovy/nextflow/Session.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/Session.groovy @@ -1209,6 +1209,11 @@ class Session implements ISession { if( aborted || cancelled || error ) return + if( workDir.scheme != 'file' ) { + log.warn "The `cleanup` option is not supported for remote work directory: ${workDir.toUriString()}" + return + } + log.trace "Cleaning-up workdir" try (CacheDB db = CacheFactory.create(uniqueId, runName).openForRead()) { db.eachRecord { HashCode hash, TraceRecord record -> From 996f557834e7a47da50f96669a2c87a3b0611bb3 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sat, 8 Feb 2025 21:20:06 +0100 Subject: [PATCH 57/66] Fix tests Signed-off-by: Paolo Di Tommaso --- .../groovy/nextflow/extension/op/Op.groovy | 2 + .../nextflow/splitter/AbstractSplitter.groovy | 12 +-- .../splitter/BytesSplitterTest.groovy | 18 ++--- .../splitter/FastaSplitterTest.groovy | 31 ++------ .../splitter/FastqSplitterTest.groovy | 20 +---- .../splitter/StringSplitterTest.groovy | 22 ++---- .../nextflow/splitter/TextSplitterTest.groovy | 73 +++++-------------- 7 files changed, 47 insertions(+), 131 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy index a4bed06c08..7391a4b461 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy @@ -66,6 +66,8 @@ class Op { } static void bind(DataflowProcessor dp, DataflowWriteChannel channel, Object msg) { + if( dp==null ) + throw new IllegalStateException("DataflowProcessor argument cannot be null") try { if( msg instanceof PoisonPill ) { channel.bind(msg) diff --git a/modules/nextflow/src/main/groovy/nextflow/splitter/AbstractSplitter.groovy b/modules/nextflow/src/main/groovy/nextflow/splitter/AbstractSplitter.groovy index 8d1263516e..24bead1cce 100644 --- a/modules/nextflow/src/main/groovy/nextflow/splitter/AbstractSplitter.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/splitter/AbstractSplitter.groovy @@ -429,12 +429,14 @@ abstract class AbstractSplitter implements SplitterStrategy { protected void append( into, value ) { log.trace "Splitter value: ${debugCount++}" - if( into instanceof Collection ) + if( into instanceof Collection ) { into.add(value) - - else if( into instanceof DataflowWriteChannel ) - Op.bind(processor, into, value) - + } + else if( into instanceof DataflowWriteChannel ) { + processor!=null + ? Op.bind(processor, into, value) + : into.bind(value) + } else throw new IllegalArgumentException("Not a valid 'into' target object: ${into?.class?.name}") } diff --git a/modules/nextflow/src/test/groovy/nextflow/splitter/BytesSplitterTest.groovy b/modules/nextflow/src/test/groovy/nextflow/splitter/BytesSplitterTest.groovy index d7986392df..e572451067 100644 --- a/modules/nextflow/src/test/groovy/nextflow/splitter/BytesSplitterTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/splitter/BytesSplitterTest.groovy @@ -30,36 +30,28 @@ class BytesSplitterTest extends Specification { def testSplitterCount() { expect: new BytesSplitter().options(by: 6).target(bytes).count() == 3 - } - def testSplitterList() { - expect: new BytesSplitter().options(by: 5).target(bytes).list() == [ [0, 1, 2, 3, 4] as byte[], [5, 6, 7, 8, 9] as byte[] , [ 0, 1, 2, 3, 4] as byte[], [5, 6] as byte[] ] - } def testSplitterWithLimit() { - expect: new BytesSplitter().options(by: 5, limit: 10).target(bytes).list() == [ [0, 1, 2, 3, 4] as byte[], [5, 6, 7, 8, 9] as byte[] ] new BytesSplitter().options(by: 5, limit: 13).target(bytes).list() == [ [0, 1, 2, 3, 4] as byte[], [5, 6, 7, 8, 9] as byte[], [ 0, 1, 2 ] as byte[] ] - } def testSplitterChannel() { - when: def c = new BytesSplitter().options(by: 5).target(bytes).channel() then: - c.val == [0, 1, 2, 3, 4] as byte[] - c.val == [5, 6, 7, 8, 9] as byte[] - c.val == [ 0, 1, 2, 3, 4] as byte[] - c.val == [5, 6] as byte[] - c.val == Channel.STOP - + c.unwrap() == [0, 1, 2, 3, 4] as byte[] + c.unwrap() == [5, 6, 7, 8, 9] as byte[] + c.unwrap() == [ 0, 1, 2, 3, 4] as byte[] + c.unwrap() == [5, 6] as byte[] + c.unwrap() == Channel.STOP } } diff --git a/modules/nextflow/src/test/groovy/nextflow/splitter/FastaSplitterTest.groovy b/modules/nextflow/src/test/groovy/nextflow/splitter/FastaSplitterTest.groovy index dcfde3de6d..d1e5363e3f 100644 --- a/modules/nextflow/src/test/groovy/nextflow/splitter/FastaSplitterTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/splitter/FastaSplitterTest.groovy @@ -26,7 +26,6 @@ import test.TestHelper */ class FastaSplitterTest extends Specification { - def testFastaRecord() { def fasta = / ; @@ -61,7 +60,6 @@ class FastaSplitterTest extends Specification { } - def testSplitFasta () { when: @@ -83,14 +81,12 @@ class FastaSplitterTest extends Specification { then: count == 2 - q.val == ">prot1\nLCLYTHIGRNIYYGS1\nEWIWGGFSVDKATLN\n" - q.val == ">prot2\nLLILILLLLLLALLS\nGLMPFLHTSKHRSMM\nIENY\n" - q.val == Channel.STOP - + q.unwrap() == ">prot1\nLCLYTHIGRNIYYGS1\nEWIWGGFSVDKATLN\n" + q.unwrap() == ">prot2\nLLILILLLLLLALLS\nGLMPFLHTSKHRSMM\nIENY\n" + q.unwrap() == Channel.STOP } def testSplitFastaRecord() { - given: def fasta = """\ >1aboA @@ -110,15 +106,13 @@ class FastaSplitterTest extends Specification { def q = new FastaSplitter().options(record: [id:true, seqString:true]).target(fasta) .channel() then: - q.val == [id:'1aboA', seqString: 'NLFVALYDFVASGDNTLSITKGEKLRVLGYNHNGEWCEAQTKNGQGWVPSNYITPVN'] - q.val == [id:'1ycsB', seqString: 'KGVIYALWDYEPQNDDELPMKEGDCMTIIHREDEDEIEWWWARLNDKEGYVPRNLLGLYP'] - q.val == [id:'1pht', seqString: 'GYQYRALYDYKKEREEDIDLHLGDILTVNKGSLVALGFSDGQEARPEEIGWLNGYNETTGERGDFPGTYVEYIGRKKISP'] - q.val == Channel.STOP - + q.unwrap() == [id:'1aboA', seqString: 'NLFVALYDFVASGDNTLSITKGEKLRVLGYNHNGEWCEAQTKNGQGWVPSNYITPVN'] + q.unwrap() == [id:'1ycsB', seqString: 'KGVIYALWDYEPQNDDELPMKEGDCMTIIHREDEDEIEWWWARLNDKEGYVPRNLLGLYP'] + q.unwrap() == [id:'1pht', seqString: 'GYQYRALYDYKKEREEDIDLHLGDILTVNKGSLVALGFSDGQEARPEEIGWLNGYNETTGERGDFPGTYVEYIGRKKISP'] + q.unwrap() == Channel.STOP } def testSplitFastaFile () { - setup: def file = File.createTempFile('chunk','test') file.deleteOnExit() @@ -148,7 +142,6 @@ class FastaSplitterTest extends Specification { result[1] == ">prot3\nDD\n>prot4\nEE\nFF\nGG\n" result[2] == ">prot5\nLL\nNN\n" - when: def result2 = new FastaSplitter() .options(record: [id: true, seqString: true], each:{ [ it.id, it.seqString.size() ]} ) @@ -162,12 +155,10 @@ class FastaSplitterTest extends Specification { result2[3] == [ 'prot4', 6 ] result2[4] == [ 'prot5', 4 ] result2.size() == 5 - } def testSplitWithLimit() { - given: def fasta = ''' >1aboA @@ -194,12 +185,9 @@ class FastaSplitterTest extends Specification { result[0] == [id: '1aboA'] result[1] == [id: '1ycsB'] result[2] == [id: '1pht'] - } - def testSplitToFile() { - given: def folder = TestHelper.createInMemTempDir() def fasta = ''' @@ -254,7 +242,6 @@ class FastaSplitterTest extends Specification { } def testSplitToFileByOne() { - given: def folder = TestHelper.createInMemTempDir() def fasta = ''' @@ -299,7 +286,6 @@ class FastaSplitterTest extends Specification { def testSplitRecordBy2() { - given: def fasta = ''' >1aboA @@ -326,7 +312,6 @@ class FastaSplitterTest extends Specification { result[0] == [[id: '1aboA'], [id: '1ycsB']] result[1] == [[id: '1pht'], [id: '1vie']] result[2] == [[id: '1ihvA']] - } def 'should split by size' () { @@ -375,7 +360,6 @@ class FastaSplitterTest extends Specification { } def 'should fetch a record' () { - given: def fasta = """\ >prot1 @@ -415,7 +399,6 @@ class FastaSplitterTest extends Specification { splitter.counter.increment == 31 splitter.counter.size == 1024 * 1024 - } } diff --git a/modules/nextflow/src/test/groovy/nextflow/splitter/FastqSplitterTest.groovy b/modules/nextflow/src/test/groovy/nextflow/splitter/FastqSplitterTest.groovy index 8abe0123c6..6983ff0423 100644 --- a/modules/nextflow/src/test/groovy/nextflow/splitter/FastqSplitterTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/splitter/FastqSplitterTest.groovy @@ -25,7 +25,6 @@ import test.TestHelper class FastqSplitterTest extends Specification { def testFastqRead() { - given: def text = ''' @HWI-EAS209_0006_FC706VJ:5:58:5894:21141#ATCACG/1 @@ -53,7 +52,6 @@ class FastqSplitterTest extends Specification { entries[0].readString == 'TTAATTGGTAAATAAATCTCCTAATAGCTTAGATNTTACCTTNNNNNNNNNNTAGTTTCTTGAGATTTGTTGGGGGAGACATTTTTGTGATTGCCTTGAT' entries[0].qualityHeader == 'HWI-EAS209_0006_FC706VJ:5:58:5894:21141#ATCACG/1' entries[0].qualityString == 'efcfffffcfeefffcffffffddf`feed]`]_Ba_^__[YBBBBBBBBBBRTT\\]][]dddd`ddd^dddadd^BBBBBBBBBBBBBBBBBBBBBBBB' - } static final FASTAQ1 = ''' @@ -118,7 +116,6 @@ class FastqSplitterTest extends Specification { '''.stripIndent().trim() def testFastqSplitByRecord() { - when: def records = new FastqSplitter().target(FASTQ2).options(record: true).list() then: @@ -156,7 +153,6 @@ class FastqSplitterTest extends Specification { records[2].readString == 'CGGGGAGCGCGGGCCCGGCAGCAGGATGATGCTCTCCCGGGCCAAGCCGGCTGTAGGGAGCACCCCGCCGCAGGGGGACAGGCGAGATCGGAAGAGCACACGTCT' records[2].qualityHeader == '' records[2].qualityString == 'BCCFFDFFHHHHHJJJJJIJHHHHFFFFEEEEEEEDDDDDDBDBDBBDBBDBBB(:ABCDDDDDDDDDDDDDDDD@BBBDDDDDDDDDDDDBDDDDDDDDDDADC' - } @@ -193,7 +189,6 @@ class FastqSplitterTest extends Specification { .stripIndent().trim() def testFastqSplitBy3() { - when: def items = new FastqSplitter().target(FASTQ3).options(by:3).list() @@ -237,18 +232,15 @@ class FastqSplitterTest extends Specification { CCCFFFFFHHHGHJJJJJJJJJHBB?BDDD5:1:6@DDBBD@D68@<CCDDCDDDCDDBBD>@?AA?B@D55@@BBD### ''' .stripIndent().leftTrim() - } def testFastqSplitBy3ToRecord() { - when: def items = new FastqSplitter().target(FASTQ3).options(by:3, record:[readHeader:true]).list() then: items[0] == [ [readHeader: 'SRR636272.19519409/1'], [readHeader: 'SRR636272.13995011/1'], [readHeader: 'SRR636272.21107783/1']] items[1] == [ [readHeader: 'SRR636272.23331539/1'], [readHeader: 'SRR636272.7306321/1'], [readHeader: 'SRR636272.23665592/1']] items[2] == [ [readHeader: 'SRR636272.1267179/1'] ] - } def testFastqSplitBy3ToFile() { @@ -297,11 +289,9 @@ class FastqSplitterTest extends Specification { CCCFFFFFHHHGHJJJJJJJJJHBB?BDDD5:1:6@DDBBD@D68@<CCDDCDDDCDDBBD>@?AA?B@D55@@BBD### ''' .stripIndent().leftTrim() - } def testFastqSplitWithLimit() { - when: def items = new FastqSplitter().options(limit:3, record:[readHeader:true]).target(FASTQ3).list() @@ -310,7 +300,6 @@ class FastqSplitterTest extends Specification { items[0] == [readHeader: 'SRR636272.19519409/1'] items[1] == [readHeader: 'SRR636272.13995011/1'] items[2] == [readHeader: 'SRR636272.21107783/1'] - } @@ -374,9 +363,8 @@ class FastqSplitterTest extends Specification { .stripIndent().trim() - def testQualityCheckWithString () { - + given: FastqSplitter splitter when: @@ -388,12 +376,9 @@ class FastqSplitterTest extends Specification { splitter = new FastqSplitter().target(fastq64) then: splitter.qualityScore() == 64 - } - def testQualityCheckWithPath () { - given: def file = TestHelper.createInMemTempFile() file.text = fastq64 @@ -402,10 +387,8 @@ class FastqSplitterTest extends Specification { FastqSplitter splitter = new FastqSplitter().target(file) then: splitter.qualityScore() == 64 - } - def testDetectFastqQuality() { given: def quality33 = "CCCFFFFDHHD;FF=GGDHGGHIIIGHIIIBDGBFCAHG@E=6?CBDBB;?BB@BD8BB;BDB<>>;@?BB<9>&5 Date: Sat, 8 Feb 2025 23:54:09 +0100 Subject: [PATCH 58/66] Add splitFastq operator Signed-off-by: Paolo Di Tommaso --- .../groovy/nextflow/extension/IntoOp.groovy | 52 +++---- .../groovy/nextflow/extension/SplitOp.groovy | 12 +- .../extension/SplitterMergeClosure.groovy | 4 +- .../extension/op/ContextSequential.groovy | 6 + .../groovy/nextflow/processor/TaskRun.groovy | 1 - .../main/groovy/nextflow/prov/Tracker.groovy | 27 ++-- .../nextflow/splitter/FastqSplitter.groovy | 6 +- .../test/groovy/nextflow/prov/ProvTest.groovy | 139 ++++++++++++++++-- .../groovy/test/TestHelper.groovy | 2 +- 9 files changed, 184 insertions(+), 65 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/IntoOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/IntoOp.groovy index 17de3dac12..47d7cb974e 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/IntoOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/IntoOp.groovy @@ -31,6 +31,9 @@ import nextflow.Global import nextflow.NF import nextflow.Session import static nextflow.extension.DataflowHelper.newChannelBy + +import nextflow.extension.op.Op + /** * Implements the {@link OperatorImpl#into} operators logic * @@ -68,46 +71,29 @@ class IntoOp { List getOutputs() { outputs } IntoOp apply() { - - final params = [:] - params.inputs = [source] - params.outputs = outputs - params.listeners = createListener() - - DataflowHelper.newOperator(params, new ChainWithClosure(new CopyChannelsClosure())) - + new SubscribeOp() + .withSource(source) + .withOnNext(this.&doNext) + .withOnComplete(this.&doComplete) + .apply() return this } - private createListener() { - - final stopOnFirst = source instanceof DataflowExpression - final listener = new DataflowEventAdapter() { - @Override - void afterRun(DataflowProcessor dp, List messages) { - if( !stopOnFirst ) return - // -- terminate the process - dp.terminate() - // -- close the output channels - for( def it : outputs ) { - if( !(it instanceof DataflowExpression)) - it.bind(Channel.STOP) - - else if( !(it as DataflowExpression).isBound() ) - it.bind(Channel.STOP) + private void doNext(DataflowProcessor dp, Object it) { + for( DataflowWriteChannel ch : outputs ) { + Op.bind(dp, ch, it) + } + } - } + private void doComplete(DataflowProcessor dp) { + for( DataflowWriteChannel ch : outputs ) { + if( ch instanceof DataflowExpression ) { + if( !ch.isBound()) Op.bind(dp, ch, Channel.STOP) } - - @Override - public boolean onException(final DataflowProcessor dp, final Throwable e) { - log.error("@unknown", e) - session.abort(e) - return true; + else { + Op.bind(dp, ch, Channel.STOP) } } - - return [listener] } } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/SplitOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/SplitOp.groovy index c3360ae0da..2728cdd30a 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/SplitOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/SplitOp.groovy @@ -86,7 +86,7 @@ class SplitOp { this.methodName = methodName if( params.pe && methodName != 'splitFastq' ) - throw new IllegalArgumentException("Unknown argument 'pe' for operator 'splitFastq'") + throw new IllegalArgumentException("Unknown argument 'pe' for operator '${methodName}'") if( params.pe==true && params.elem ) throw new IllegalArgumentException("Parameter `pe` and `elem` conflicts") @@ -109,7 +109,6 @@ class SplitOp { if( params.into && !(CH.isChannelQueue(params.into)) ) throw new IllegalArgumentException('Parameter `into` must reference a channel object') - } /** @@ -131,22 +130,22 @@ class SplitOp { final cardinality = indexes.size() // -- creates a copy of `source` channel for each element to split - def copies = createSourceCopies(source, cardinality) + final copies = createSourceCopies(source, cardinality) // -- applies the splitter the each channel copy - def splitted = new ArrayList(cardinality) + final splitted = new ArrayList(cardinality) for( int i=0; i */ +@CompileStatic +@Slf4j class ContextSequential implements OpContext { private final Map holder = new ConcurrentHashMap<>(1) @@ -35,12 +39,14 @@ class ContextSequential implements OpContext { OperatorRun allocateRun() { final result = new OperatorRun() holder.put('run', result) + log.trace "+ AllocateRun run=$result" return result } @Override OperatorRun getOperatorRun() { final result = holder.get('run') + log.trace "+ GetOperatorRun=$result" return result } } diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy index 92793f62ca..881bc26701 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy @@ -122,7 +122,6 @@ class TaskRun implements Cloneable, TrailRun { outputs[param] = value } - /** * The value to be piped to the process stdin */ diff --git a/modules/nextflow/src/main/groovy/nextflow/prov/Tracker.groovy b/modules/nextflow/src/main/groovy/nextflow/prov/Tracker.groovy index db21d2bbb7..1ecc62529b 100644 --- a/modules/nextflow/src/main/groovy/nextflow/prov/Tracker.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/prov/Tracker.groovy @@ -65,11 +65,11 @@ class Tracker { private logInputs(TaskRun task, List inputs) { if( log.isTraceEnabled() ) { def msg = "Task input" - msg += "\n - id : ${task.id} " + msg += "\n - task id : ${task.id} " msg += "\n - name : '${task.name}'" - msg += "\n - upstream: ${task.upstreamTasks*.value.join(',')}" + msg += "\n - upstream: ${task.upstreamTasks*.value}" for( Object it : inputs ) { - msg += "\n<= ${it}" + msg += "\n=> ${dumpInput(it)}" } log.trace(msg) } @@ -78,14 +78,23 @@ class Tracker { private logInputs(OperatorRun run, List inputs) { if( log.isTraceEnabled() ) { def msg = "Operator input" - msg += "\n - id: ${System.identityHashCode(run)} " + msg += "\n - run id: ${System.identityHashCode(run)} " for( Object it : inputs ) { - msg += "\n<= ${it}" + msg += "\n=> ${dumpInput(it)}" } log.trace(msg) } } + private String dumpInput(Object it) { + if( it==null ) + return "null" + if( it instanceof Msg ) + return it.toString() + else + return "${it.getClass().getName()}[id=${System.identityHashCode(it)}; value=${it.toString()}]" + } + List receiveInputs(OperatorRun run, List inputs) { // find the upstream tasks id run.inputIds.addAll(inputs.collect(msg-> System.identityHashCode(msg))) @@ -148,16 +157,16 @@ class Tracker { String str if( run instanceof OperatorRun ) { str = "Operator output" - str += "\n - id : ${System.identityHashCode(run)}" + str += "\n - run id: ${System.identityHashCode(run)}" } else if( run instanceof TaskRun ) { str = "Task output" - str += "\n - id : ${run.id}" - str += "\n - name: '${run.name}'" + str += "\n - task id : ${run.id}" + str += "\n - name : '${run.name}'" } else throw new IllegalArgumentException("Unknown run type: ${run}") - str += "\n=> ${msg}" + str += "\n<= ${msg}" log.trace(str) } } diff --git a/modules/nextflow/src/main/groovy/nextflow/splitter/FastqSplitter.groovy b/modules/nextflow/src/main/groovy/nextflow/splitter/FastqSplitter.groovy index 7cdd9e5c80..83e41907dd 100644 --- a/modules/nextflow/src/main/groovy/nextflow/splitter/FastqSplitter.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/splitter/FastqSplitter.groovy @@ -59,7 +59,7 @@ class FastqSplitter extends AbstractTextSplitter { @Override protected Map validOptions() { - def result = super.validOptions() + final result = super.validOptions() result.record = [ Boolean, Map ] return result } @@ -83,7 +83,7 @@ class FastqSplitter extends AbstractTextSplitter { } static Map recordToMap( String l1, String l2, String l3, String l4, Map fields ) { - def result = [:] + final result = [:] if( !fields || fields.containsKey('readHeader')) result.readHeader = l1.substring(1) @@ -161,7 +161,7 @@ class FastqSplitter extends AbstractTextSplitter { * * @param quality A fastq quality string */ - def int qualityScore(Map opts=null) { + int qualityScore(Map opts=null) { if( opts ) options(opts) diff --git a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy index d2781a85a7..bfca0e1a13 100644 --- a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy @@ -1094,7 +1094,7 @@ class ProvTest extends Dsl2Spec { when: dsl_eval(globalConfig(), """ workflow { - p1 | splitText | p2 + p1 | splitText(file:true) | p2 } process p1 { @@ -1122,7 +1122,125 @@ class ProvTest extends Dsl2Spec { } - def 'should track provenance with splitFastq paired'() { + def 'should track provenance with splitFasta and a file'() { + given: + def FASTA = """\ + >1aboA + NLFVALYDFVASGDNTLSITKGEKLRVLGYNHNGEWCEAQTKNGQGWVPS + NYITPVN + >1ycsB + KGVIYALWDYEPQNDDELPMKEGDCMTIIHREDEDEIEWWWARLNDKEGY + VPRNLLGLYP + >1pht + GYQYRALYDYKKEREEDIDLHLGDILTVNKGSLVALGFSDGQEARPEEIG + WLNGYNETTGERGDFPGTYVE + YIGRKKISP + """.stripIndent() + + when: + dsl_eval(globalConfig(), """ + workflow { + p1 | splitFasta(file:true) | p2 + } + + process p1 { + output: path('result.txt') + exec: + task.workDir.resolve('result.txt').text = '''${FASTA}''' + } + + process p2 { + input: val(chunk) + exec: + println "\${task.name}: chunck=\$chunk" + } + """) + + then: + upstreamTasksOf('p2 (1)') + .name == ['p1'] + and: + upstreamTasksOf('p2 (2)') + .name == ['p1'] + and: + upstreamTasksOf('p2 (3)') + .name == ['p1'] + + } + + def 'should track provenance with splitFastq and a file'() { + when: + dsl_eval(globalConfig(), """ + workflow { + p1 | splitFastq(file:true) | p2 + } + + process p1 { + output: path('result.txt') + exec: + task.workDir.resolve('result.txt').text = '''${SplitFastqOp2Test.READS1}''' + } + + process p2 { + input: val(chunk) + exec: + println "\${task.name}: chunck=\$chunk" + } + """) + + then: + upstreamTasksOf('p2 (1)') + .name == ['p1'] + and: + upstreamTasksOf('p2 (2)') + .name == ['p1'] + and: + upstreamTasksOf('p2 (3)') + .name == ['p1'] + and: + upstreamTasksOf('p2 (4)') + .name == ['p1'] + } + + def 'should track provenance with splitFastq with paired files'() { + when: + dsl_eval(globalConfig(), """ + workflow { + p1() + p1.out | splitFastq(pe:true, file:true) | p2 + } + + process p1 { + output: tuple val(sample), path('one.fq'), path('two.fq') + exec: + sample = 'sample_1' + task.workDir.resolve('one.fq').text = '''${SplitFastqOp2Test.READS1}''' + task.workDir.resolve('two.fq').text = '''${SplitFastqOp2Test.READS2}''' + } + process p2 { + input: tuple val(sample), path(r1), path(r2) + exec: + println "\${task.name}: sample=\${sample};" + } + + """) + + then: + noExceptionThrown() + upstreamTasksOf('p2 (1)') + .name == ['p1'] + and: + upstreamTasksOf('p2 (2)') + .name == ['p1'] + and: + upstreamTasksOf('p2 (3)') + .name == ['p1'] + and: + upstreamTasksOf('p2 (4)') + .name == ['p1'] + } + + def 'should track provenance with splitFastq join and paired files'() { given: when: @@ -1158,13 +1276,16 @@ class ProvTest extends Dsl2Spec { then: noExceptionThrown() upstreamTasksOf('p3 (1)') - .name == ['p1 (1)', 'p2 (1)'] -// and: -// upstreamTasksOf('p2 (2)') -// .name == ['p1'] -// and: -// upstreamTasksOf('p2 (3)') -// .name == ['p1'] + .id == [TaskId.of(1), TaskId.of(2)] + and: + upstreamTasksOf('p3 (2)') + .id == [TaskId.of(1), TaskId.of(2)] + and: + upstreamTasksOf('p3 (3)') + .id == [TaskId.of(1), TaskId.of(2)] + and: + upstreamTasksOf('p3 (4)') + .id == [TaskId.of(1), TaskId.of(2)] } diff --git a/modules/nextflow/src/testFixtures/groovy/test/TestHelper.groovy b/modules/nextflow/src/testFixtures/groovy/test/TestHelper.groovy index 8ca3fcb841..f47a72bc1c 100644 --- a/modules/nextflow/src/testFixtures/groovy/test/TestHelper.groovy +++ b/modules/nextflow/src/testFixtures/groovy/test/TestHelper.groovy @@ -116,7 +116,7 @@ class TestHelper { static List upstreamTasksOf(TaskRun t) { final ids = t.upstreamTasks ?: Set.of() - return ids.collect(it -> getTaskById(it)) + return ids.collect(it -> getTaskById(it)).sort((run)->run.id) } static TaskRun getTaskByName(String name) { From 63c5440d36169d2013a5c8cc85922b87abfea1e6 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sun, 9 Feb 2025 12:22:37 +0100 Subject: [PATCH 59/66] Remove unused chain operator Signed-off-by: Paolo Di Tommaso --- .../groovy/nextflow/extension/ChainOp.groovy | 86 ------------------- 1 file changed, 86 deletions(-) delete mode 100644 modules/nextflow/src/main/groovy/nextflow/extension/ChainOp.groovy diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/ChainOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/ChainOp.groovy deleted file mode 100644 index 3550c28fc8..0000000000 --- a/modules/nextflow/src/main/groovy/nextflow/extension/ChainOp.groovy +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright 2013-2024, Seqera Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package nextflow.extension - -import groovy.transform.CompileStatic -import groovyx.gpars.dataflow.DataflowReadChannel -import groovyx.gpars.dataflow.DataflowWriteChannel -import groovyx.gpars.dataflow.operator.ChainWithClosure -import groovyx.gpars.dataflow.operator.DataflowEventListener -import nextflow.extension.op.Op -/** - * Implements the chain operator - * - * @author Paolo Di Tommaso - */ -@CompileStatic -class ChainOp { - - private DataflowReadChannel source - private DataflowWriteChannel target - private List listeners = List.of() - private Closure action - - static ChainOp create() { - new ChainOp() - } - - ChainOp withSource(DataflowReadChannel source) { - assert source - this.source = source - return this - } - - ChainOp withTarget(DataflowWriteChannel target) { - assert target - this.target = target - return this - } - - ChainOp withListener(DataflowEventListener listener) { - assert listener != null - this.listeners = List.of(listener) - return this - } - - ChainOp withListeners(List listeners) { - assert listeners != null - this.listeners = listeners - return this - } - - ChainOp withAction(Closure action) { - this.action = action - return this - } - - DataflowWriteChannel apply() { - assert source - assert target - assert action - - new Op() - .withInput(source) - .withOutput(target) - .withListeners(listeners) - .withCode(new ChainWithClosure(action)) - .apply() - - return target - } -} From 662b467b0d5a369eca759db44dd2d2f7e8f0d9be Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sun, 9 Feb 2025 14:05:33 +0100 Subject: [PATCH 60/66] Into op cleanup Signed-off-by: Paolo Di Tommaso --- .../main/groovy/nextflow/extension/IntoOp.groovy | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/IntoOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/IntoOp.groovy index 47d7cb974e..5afe4d1e96 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/IntoOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/IntoOp.groovy @@ -22,20 +22,11 @@ import groovyx.gpars.dataflow.DataflowQueue import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowWriteChannel import groovyx.gpars.dataflow.expression.DataflowExpression -import groovyx.gpars.dataflow.operator.ChainWithClosure -import groovyx.gpars.dataflow.operator.CopyChannelsClosure -import groovyx.gpars.dataflow.operator.DataflowEventAdapter import groovyx.gpars.dataflow.operator.DataflowProcessor import nextflow.Channel -import nextflow.Global -import nextflow.NF -import nextflow.Session -import static nextflow.extension.DataflowHelper.newChannelBy - import nextflow.extension.op.Op - /** - * Implements the {@link OperatorImpl#into} operators logic + * Implements the "into" operator logic * * @author Paolo Di Tommaso */ @@ -47,8 +38,6 @@ class IntoOp { private List outputs - private Session session = (Session)Global.session - IntoOp( DataflowReadChannel source, List targets ) { assert source assert targets From 40b900f60f830f55d3739f4e0b5d41ce55d46c69 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sun, 9 Feb 2025 14:29:52 +0100 Subject: [PATCH 61/66] Remove deprecated code Signed-off-by: Paolo Di Tommaso --- .../nextflow/extension/DataflowHelper.groovy | 132 ++---------------- .../groovy/nextflow/extension/FilterOp.groovy | 27 ++-- .../groovy/nextflow/extension/MapOp.groovy | 32 +++-- .../groovy/nextflow/extension/MergeOp.groovy | 14 +- .../nextflow/extension/OperatorImpl.groovy | 8 +- .../groovy/nextflow/extension/TapOp.groovy | 8 +- .../nextflow/extension/UntilManyOp.groovy | 9 +- .../groovy/nextflow/extension/op/Op.groovy | 4 + 8 files changed, 72 insertions(+), 162 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy index 5ed5951cdb..da2b84ea77 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy @@ -21,10 +21,7 @@ import java.lang.reflect.InvocationTargetException import groovy.transform.CompileStatic import groovy.transform.PackageScope import groovy.util.logging.Slf4j -import groovyx.gpars.dataflow.DataflowChannel -import groovyx.gpars.dataflow.DataflowQueue import groovyx.gpars.dataflow.DataflowReadChannel -import groovyx.gpars.dataflow.DataflowVariable import groovyx.gpars.dataflow.DataflowWriteChannel import groovyx.gpars.dataflow.expression.DataflowExpression import groovyx.gpars.dataflow.operator.DataflowEventAdapter @@ -45,29 +42,6 @@ class DataflowHelper { private static Session getSession() { Global.getSession() as Session } - /** - * Create a dataflow object by the type of the specified source argument - * - * @param source - * @return - */ - @Deprecated - static DataflowChannel newChannelBy(DataflowReadChannel source) { - - switch( source ) { - case DataflowExpression: - return new DataflowVariable() - - case DataflowQueue: - return new DataflowQueue() - - default: - throw new IllegalArgumentException() - } - - } - - /** * Check if a {@code DataflowProcessor} is active * @@ -88,20 +62,19 @@ class DataflowHelper { /* * The default operators listener when no other else is specified */ - @PackageScope - static DEF_ERROR_LISTENER = new DataflowEventAdapter() { - @Override - boolean onException(final DataflowProcessor dp, final Throwable t) { - final e = t instanceof InvocationTargetException ? t.cause : t - OperatorImpl.log.error("@unknown", e) - session?.abort(e) - return true; + static DataflowEventListener defaultErrorListener() { + return new DataflowEventAdapter() { + @Override + boolean onException(final DataflowProcessor dp, final Throwable t) { + final e = t instanceof InvocationTargetException ? t.cause : t + OperatorImpl.log.error("@unknown", e) + session?.abort(e) + return true; + } } } - @PackageScope - static DataflowEventAdapter stopErrorListener(DataflowReadChannel source, DataflowWriteChannel target) { - + static DataflowEventListener stopErrorListener(DataflowReadChannel source, DataflowWriteChannel target) { new DataflowEventAdapter() { @Override void afterRun(final DataflowProcessor dp, final List messages) { @@ -119,91 +92,6 @@ class DataflowHelper { return true } } - - } - - @PackageScope - static Map createOpParams(inputs, outputs, listeners) { - final params = new HashMap(3) - params.inputs = inputs instanceof List ? inputs : [inputs] - params.outputs = outputs instanceof List ? outputs : [outputs] - params.listeners = listeners instanceof List ? listeners : [listeners] - return params - } - - /** - * Creates a new {@code Dataflow.operator} adding the created instance to the current session list - * - * @see nextflow.Session#allOperators - * - * @param params The map holding inputs, outputs channels and other parameters - * @param code The closure to be executed by the operator - */ - @Deprecated - static DataflowProcessor newOperator( Map params, Closure code ) { - - // -- add a default error listener - if( !params.listeners ) { - // add the default error handler - params.listeners = [ DEF_ERROR_LISTENER ] - } - - return new Op() - .withParams(params) - .withCode(code) - .apply() - } - - /** - * Creates a new {@code Dataflow.operator} adding the created instance to the current session list - * - * @see nextflow.Session#allOperators - * - * @param inputs The list of the input {@code DataflowReadChannel}s - * @param outputs The list of list output {@code DataflowWriteChannel}s - * @param code The closure to be executed by the operator - */ - @Deprecated - static DataflowProcessor newOperator( List inputs, List outputs, Closure code ) { - newOperator( inputs: inputs, outputs: outputs, code ) - } - - /** - * Creates a new {@code Dataflow.operator} adding the created instance to the current session list - * - * @see nextflow.Session#allOperators - * - * @param input An instance of {@code DataflowReadChannel} representing the input channel - * @param output An instance of {@code DataflowWriteChannel} representing the output channel - * @param code The closure to be executed by the operator - */ - static DataflowProcessor newOperator( DataflowReadChannel input, DataflowWriteChannel output, Closure code ) { - newOperator(input, output, DEF_ERROR_LISTENER, code ) - } - - /** - * Creates a new {@code Dataflow.operator} adding the created instance to the current session list - * - * @see nextflow.Session#allOperators - * - * @param input An instance of {@code DataflowReadChannel} representing the input channel - * @param output An instance of {@code DataflowWriteChannel} representing the output channel - * @param listener An instance of {@code DataflowEventListener} listening to operator's events - * @param code The closure to be executed by the operator - */ - static DataflowProcessor newOperator( DataflowReadChannel input, DataflowWriteChannel output, DataflowEventListener listener, Closure code ) { - if( !listener ) - listener = DEF_ERROR_LISTENER - - final params = [:] - params.inputs = [input] - params.outputs = [output] - params.listeners = [listener] - - return new Op() - .withParams(params) - .withCode(code) - .apply() } static final DataflowProcessor subscribeImpl(final DataflowReadChannel source, final Map events ) { diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/FilterOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/FilterOp.groovy index df5bec5842..d736deef98 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/FilterOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/FilterOp.groovy @@ -17,7 +17,6 @@ package nextflow.extension -import static nextflow.extension.DataflowHelper.* import groovy.transform.CompileStatic import groovyx.gpars.dataflow.DataflowReadChannel @@ -64,18 +63,22 @@ class FilterOp { : null final target = CH.createBy(source) final stopOnFirst = source instanceof DataflowExpression - newOperator(source, target, { - final result = criteria instanceof Closure - ? DefaultTypeTransformation.castToBoolean(criteria.call(it)) - : discriminator.invoke(criteria, (Object)it) - final dp = getDelegate() as DataflowProcessor - if( result ) { - Op.bind(dp, target, it) + new Op() + .withInput(source) + .withOutput(target) + .withCode { + final result = criteria instanceof Closure + ? DefaultTypeTransformation.castToBoolean(criteria.call(it)) + : discriminator.invoke(criteria, (Object)it) + final dp = getDelegate() as DataflowProcessor + if( result ) { + Op.bind(dp, target, it) + } + if( stopOnFirst ) { + Op.bind(dp, target, Channel.STOP) + } } - if( stopOnFirst ) { - Op.bind(dp, target, Channel.STOP) - } - }) + .apply() return target } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/MapOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/MapOp.groovy index 87b0e95306..bd4577c01d 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/MapOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/MapOp.groovy @@ -68,21 +68,23 @@ class MapOp { target = CH.createBy(source) final stopOnFirst = source instanceof DataflowExpression - DataflowHelper.newOperator(source, target) { it -> - - final result = mapper.call(it) - final proc = getDelegate() as DataflowProcessor - - // bind the result value - if (result != Channel.VOID) - Op.bind(proc, target, result) - - // when the `map` operator is applied to a dataflow flow variable - // terminate the processor after the first emission -- Issue #44 - if( stopOnFirst ) - proc.terminate() - - } + new Op() + .withInput(source) + .withOutput(target) + .withCode { it -> + final result = mapper.call(it) + final proc = getDelegate() as DataflowProcessor + + // bind the result value + if (result != Channel.VOID) + Op.bind(proc, target, result) + + // when the `map` operator is applied to a dataflow flow variable + // terminate the processor after the first emission -- Issue #44 + if( stopOnFirst ) + proc.terminate() + } + .apply() return target } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/MergeOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/MergeOp.groovy index 561567575b..ad4f9f04fc 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/MergeOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/MergeOp.groovy @@ -17,15 +17,13 @@ package nextflow.extension -import static nextflow.extension.DataflowHelper.createOpParams -import static nextflow.extension.DataflowHelper.newOperator -import static nextflow.extension.DataflowHelper.stopErrorListener +import static nextflow.extension.DataflowHelper.* import groovy.transform.CompileStatic import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowWriteChannel import groovyx.gpars.dataflow.operator.ChainWithClosure - +import nextflow.extension.op.Op /** * Implements merge operator * @@ -56,8 +54,12 @@ class MergeOp { inputs.add(source) inputs.addAll(others) final listener = stopErrorListener(source,result) - final params = createOpParams(inputs, result, listener) - newOperator(params, action) + new Op() + .withInputs(inputs) + .withOutput(result) + .withListener(listener) + .withCode(action) + .apply() return result } } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy index 86d074bf67..8e2539d058 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy @@ -802,8 +802,12 @@ class OperatorImpl { final inputs = [source, other] final action = closure ? new ChainWithClosure<>(closure) : new DefaultMergeClosure(inputs.size()) final listener = stopErrorListener(source,result) - final params = createOpParams(inputs, result, listener) - newOperator(params, action) + new Op() + .withInputs(inputs) + .withOutput(result) + .withListener(listener) + .withCode(action) + .apply() return result; } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/TapOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/TapOp.groovy index 15d8527c51..e75b029e33 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/TapOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/TapOp.groovy @@ -23,7 +23,7 @@ import groovyx.gpars.dataflow.DataflowWriteChannel import groovyx.gpars.dataflow.operator.ChainWithClosure import groovyx.gpars.dataflow.operator.CopyChannelsClosure import nextflow.NF -import static nextflow.extension.DataflowHelper.newOperator +import nextflow.extension.op.Op /** * Implements the {@link OperatorImpl#tap} operator * @@ -108,7 +108,11 @@ class TapOp { * @return An instance of {@link TapOp} itself */ TapOp apply() { - newOperator([source], outputs, new ChainWithClosure(new CopyChannelsClosure())); + new Op() + .withInput(source) + .withOutputs(outputs) + .withCode(new ChainWithClosure(new CopyChannelsClosure())) + .apply() return this } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/UntilManyOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/UntilManyOp.groovy index 42f05d8ca6..98cfe60afc 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/UntilManyOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/UntilManyOp.groovy @@ -17,12 +17,12 @@ package nextflow.extension -import static nextflow.extension.DataflowHelper.* import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowWriteChannel import groovyx.gpars.dataflow.operator.DataflowProcessor import nextflow.Channel +import nextflow.extension.op.Op import org.codehaus.groovy.runtime.InvokerHelper import org.codehaus.groovy.runtime.typehandling.DefaultTypeTransformation /** @@ -46,8 +46,11 @@ class UntilManyOp { for( DataflowReadChannel it : sources ) targets << CH.createBy(it) - newOperator(sources, targets, new UntilCondition(len, closure)) - + new Op() + .withInputs(sources) + .withOutputs(targets) + .withCode(new UntilCondition(len, closure)) + .apply() return targets } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy index 7391a4b461..b50e62f9fa 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy @@ -31,6 +31,7 @@ import groovyx.gpars.dataflow.operator.PoisonPill import nextflow.Global import nextflow.Session import nextflow.dag.NodeMarker +import nextflow.extension.DataflowHelper import nextflow.prov.OperatorRun import nextflow.prov.Prov import nextflow.prov.Tracker @@ -294,6 +295,9 @@ class Op { assert code assert context + if( !listeners ) + listeners = List.of(DataflowHelper.defaultErrorListener()) + // Encapsulate the target "code" closure with a "OpClosure" object // to grab input data and track the execution provenance final closure = new OpClosure(code, context) From b83e15faeb721d8c3f2145ed7299c1ce2dc08cdf Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sun, 9 Feb 2025 14:43:46 +0100 Subject: [PATCH 62/66] Name refactor Signed-off-by: Paolo Di Tommaso --- .../src/main/groovy/nextflow/processor/TaskRun.groovy | 4 ++-- .../src/main/groovy/nextflow/prov/OperatorRun.groovy | 2 +- .../nextflow/prov/{TrailRun.groovy => ProvLink.groovy} | 2 +- .../nextflow/src/main/groovy/nextflow/prov/Tracker.groovy | 6 +++--- 4 files changed, 7 insertions(+), 7 deletions(-) rename modules/nextflow/src/main/groovy/nextflow/prov/{TrailRun.groovy => ProvLink.groovy} (97%) diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy index 881bc26701..47b91474ab 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy @@ -38,7 +38,7 @@ import nextflow.exception.ProcessTemplateException import nextflow.exception.ProcessUnrecoverableException import nextflow.file.FileHelper import nextflow.file.FileHolder -import nextflow.prov.TrailRun +import nextflow.prov.ProvLink import nextflow.script.BodyDef import nextflow.script.ScriptType import nextflow.script.TaskClosure @@ -60,7 +60,7 @@ import nextflow.spack.SpackCache */ @Slf4j -class TaskRun implements Cloneable, TrailRun { +class TaskRun implements Cloneable, ProvLink { final private ConcurrentHashMap cache0 = new ConcurrentHashMap() diff --git a/modules/nextflow/src/main/groovy/nextflow/prov/OperatorRun.groovy b/modules/nextflow/src/main/groovy/nextflow/prov/OperatorRun.groovy index e3016d05c9..cda4180b1e 100644 --- a/modules/nextflow/src/main/groovy/nextflow/prov/OperatorRun.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/prov/OperatorRun.groovy @@ -27,7 +27,7 @@ import groovy.transform.CompileStatic */ @Canonical @CompileStatic -class OperatorRun implements TrailRun { +class OperatorRun implements ProvLink { /** * The list of (object) ids that was received as input by a operator run */ diff --git a/modules/nextflow/src/main/groovy/nextflow/prov/TrailRun.groovy b/modules/nextflow/src/main/groovy/nextflow/prov/ProvLink.groovy similarity index 97% rename from modules/nextflow/src/main/groovy/nextflow/prov/TrailRun.groovy rename to modules/nextflow/src/main/groovy/nextflow/prov/ProvLink.groovy index 32e1eb4b95..8b59404e48 100644 --- a/modules/nextflow/src/main/groovy/nextflow/prov/TrailRun.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/prov/ProvLink.groovy @@ -22,5 +22,5 @@ package nextflow.prov * * @author Paolo Di Tommaso */ -interface TrailRun { +interface ProvLink { } diff --git a/modules/nextflow/src/main/groovy/nextflow/prov/Tracker.groovy b/modules/nextflow/src/main/groovy/nextflow/prov/Tracker.groovy index 1ecc62529b..640aefc10f 100644 --- a/modules/nextflow/src/main/groovy/nextflow/prov/Tracker.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/prov/Tracker.groovy @@ -50,7 +50,7 @@ class Tracker { /** * Associate an output value with the corresponding task run that emitted it */ - private Map messages = new ConcurrentHashMap<>() + private Map messages = new ConcurrentHashMap<>() List receiveInputs(TaskRun task, List inputs) { // find the upstream tasks id @@ -139,7 +139,7 @@ class Tracker { return upstream } - Msg bindOutput(TrailRun run, DataflowWriteChannel channel, Object out) { + Msg bindOutput(ProvLink run, DataflowWriteChannel channel, Object out) { assert run!=null, "Argument 'run' cannot be null" assert channel!=null, "Argument 'channel' cannot be null" @@ -152,7 +152,7 @@ class Tracker { return msg } - private void logOutput(TrailRun run, Msg msg) { + private void logOutput(ProvLink run, Msg msg) { if( log.isTraceEnabled() ) { String str if( run instanceof OperatorRun ) { From 2aed5e9808f033d1cc4bebb75646e808c3529b1c Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sun, 9 Feb 2025 15:18:42 +0100 Subject: [PATCH 63/66] Add view operator [ci fast] Signed-off-by: Paolo Di Tommaso --- .../nextflow/extension/OperatorImpl.groovy | 17 +--- .../groovy/nextflow/extension/ViewOp.groovy | 79 ++++++++++++++++ .../test/groovy/nextflow/prov/ProvTest.groovy | 92 +++++++++++++++++++ 3 files changed, 174 insertions(+), 14 deletions(-) create mode 100644 modules/nextflow/src/main/groovy/nextflow/extension/ViewOp.groovy diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy index 8e2539d058..0211f02f10 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy @@ -30,9 +30,7 @@ import groovyx.gpars.dataflow.expression.DataflowExpression import groovyx.gpars.dataflow.operator.ChainWithClosure import groovyx.gpars.dataflow.operator.DataflowProcessor import nextflow.Channel -import nextflow.Global import nextflow.NF -import nextflow.Session import nextflow.extension.op.ContextRunPerThread import nextflow.extension.op.Op import nextflow.script.ChannelOut @@ -57,8 +55,6 @@ class OperatorImpl { static final public OperatorImpl instance = new OperatorImpl() - private static Session getSession() { Global.getSession() as Session } - /** * Subscribe *onNext* event * @@ -778,18 +774,11 @@ class OperatorImpl { checkParams('view', opts, PARAMS_VIEW) final newLine = opts.newLine != false - final target = CH.createBy(source); - - new SubscribeOp() + return new ViewOp() .withSource(source) - .withOnNext{ - final obj = closure != null ? closure.call(it) : it - session.printConsole(obj?.toString(), newLine) - target.bind(it) - } - .withOnComplete{ CH.close0(target) } + .withNewLine(newLine) + .withCode(closure) .apply() - return target } DataflowWriteChannel view(final DataflowReadChannel source, Closure closure = null) { diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/ViewOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/ViewOp.groovy new file mode 100644 index 0000000000..b100fa7680 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/extension/ViewOp.groovy @@ -0,0 +1,79 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.extension + +import groovy.transform.CompileStatic +import groovyx.gpars.dataflow.DataflowReadChannel +import groovyx.gpars.dataflow.DataflowWriteChannel +import groovyx.gpars.dataflow.operator.DataflowProcessor +import nextflow.Global +import nextflow.Session +import nextflow.extension.op.Op + +/** + * Implements "view" operator + * + * @author Paolo Di Tommaso + */ +@CompileStatic +class ViewOp { + + private DataflowReadChannel source + private Closure action + private boolean newLine + + private static Session getSession() { Global.getSession() as Session } + + ViewOp() { + this.source = source + this.action = action + } + + ViewOp withSource(DataflowReadChannel source) { + if( source==null ) + throw new IllegalArgumentException("Argument 'source' cannot be null") + this.source = source + return this + } + + ViewOp withCode(Closure action) { + this.action = action + return this + } + + ViewOp withNewLine(boolean newLine) { + this.newLine = newLine + return this + } + + DataflowWriteChannel apply() { + final target = CH.createBy(source); + + new SubscribeOp() + .withSource(source) + .withOnNext { DataflowProcessor dp, Object it-> + final obj = action != null ? action.call(it) : it + session.printConsole(obj?.toString(), newLine) + Op.bind(dp,target,it) + } + .withOnComplete { CH.close0(target) } + .apply() + + return target + } +} diff --git a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy index bfca0e1a13..d05ed4b448 100644 --- a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy @@ -1088,6 +1088,38 @@ class ProvTest extends Dsl2Spec { .name == ['p1 (1)'] } + def 'should track provenance with splitJson operator'() { + when: + dsl_eval(globalConfig(), ''' + workflow { + Channel.of('{"A": 1, "B": 2, "C": 3}') | p1 | splitJson | p2 + } + + process p1 { + input: val(x) + output: val(y) + exec: + y = x + } + + process p2 { + input: val(x) + exec: + println x + } + ''') + + then: + upstreamTasksOf('p2 (1)') + .name == ['p1 (1)'] + and: + upstreamTasksOf('p2 (2)') + .name == ['p1 (1)'] + and: + upstreamTasksOf('p2 (3)') + .name == ['p1 (1)'] + } + def 'should track provenance with splitText and a file'() { given: @@ -1289,4 +1321,64 @@ class ProvTest extends Dsl2Spec { } + def 'should track provenance with toInteger operator'() { + when: + dsl_eval(globalConfig(), ''' + workflow { + channel.of('1','2','3') | p1 | toInteger | p2 + } + + process p1 { + input: val(x) + output: val(y) + exec: + y = x + } + + process p2 { + input: val(x) + exec: + println x + } + ''') + + then: + upstreamTasksOf('p2 (1)') + .name == ['p1 (1)'] + and: + upstreamTasksOf('p2 (2)') + .name == ['p1 (2)'] + } + + def 'should track provenance with view operator'() { + when: + dsl_eval(globalConfig(), ''' + workflow { + channel.of('1','2','3') | p1 | view | p2 + } + + process p1 { + input: val(x) + output: val(y) + exec: + y = x + } + + process p2 { + input: val(x) + exec: + println x + } + ''') + + then: + upstreamTasksOf('p2 (1)') + .name == ['p1 (1)'] + and: + upstreamTasksOf('p2 (2)') + .name == ['p1 (2)'] + and: + upstreamTasksOf('p2 (3)') + .name == ['p1 (3)'] + } } From c7fee467676f9de3c255985f24a66e38e56c457d Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sun, 9 Feb 2025 15:47:04 +0100 Subject: [PATCH 64/66] Add random sample [ci fast] Signed-off-by: Paolo Di Tommaso --- .../nextflow/extension/RandomSampleOp.groovy | 30 +++++++++++-------- .../extension/RandomSampleTest.groovy | 4 --- .../test/groovy/nextflow/prov/ProvTest.groovy | 27 +++++++++++++++++ 3 files changed, 45 insertions(+), 16 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/RandomSampleOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/RandomSampleOp.groovy index 7b4143e0f4..3fe6f65363 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/RandomSampleOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/RandomSampleOp.groovy @@ -19,7 +19,12 @@ package nextflow.extension import groovy.transform.CompileStatic import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowWriteChannel +import groovyx.gpars.dataflow.operator.DataflowProcessor import nextflow.Channel +import nextflow.extension.op.ContextSequential +import nextflow.extension.op.Op +import nextflow.extension.op.OpContext +import nextflow.extension.op.OpDatum /** * Implements Reservoir sampling of channel content * @@ -38,46 +43,47 @@ class RandomSampleOp { private Random rng - private List reservoir = [] + private List reservoir = [] private int counter + private OpContext context = new ContextSequential() + RandomSampleOp( DataflowReadChannel source, int N, Long seed = null) { this.source = source this.N = N this.rng = seed != null ? new Random(seed) : new Random() } - - private void sampling(it) { - + private void sampling(Object it) { counter++ //Fill reservoir if (counter <= N){ - reservoir << it + reservoir.add(OpDatum.of(it, context.getOperatorRun())) } else { //Pick a random number int i = rng.nextInt(counter) if (i < N) - reservoir[i] = it + reservoir[i] = OpDatum.of(it, context.getOperatorRun()) } - } - private void emit(nop) { - + private void emit(DataflowProcessor dp) { if( counter <= N ) Collections.shuffle(reservoir, rng) - - reservoir.each { it!=null ? result.bind(it) : null } - result.bind(Channel.STOP) + for( OpDatum it : reservoir ) { + if( it!=null ) + Op.bind(it.run, result, it.value) + } + Op.bind(dp, result, Channel.STOP) } DataflowWriteChannel apply() { result = CH.create() new SubscribeOp() .withSource(source) + .withContext(context) .withOnNext(this.&sampling) .withOnComplete(this.&emit) .apply() diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/RandomSampleTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/RandomSampleTest.groovy index 6212d2cb24..c545603526 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/RandomSampleTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/RandomSampleTest.groovy @@ -31,7 +31,6 @@ class RandomSampleTest extends Specification { } def 'should produce random sample' () { - given: def ch = Channel.of('A'..'Z') def sampler = new RandomSampleOp(ch, 10) @@ -44,9 +43,7 @@ class RandomSampleTest extends Specification { result != 'A'..'J' } - def 'should produce random sample given a short channel' () { - given: def ch = Channel.of('A'..'J') def sampler = new RandomSampleOp(ch, 20) @@ -60,7 +57,6 @@ class RandomSampleTest extends Specification { } def 'should produce random sample given a channel emitting the same number of items as the buffer' () { - given: def ch = Channel.of('A'..'J') def sampler = new RandomSampleOp(ch, 10) diff --git a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy index d05ed4b448..3897b23371 100644 --- a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy @@ -1381,4 +1381,31 @@ class ProvTest extends Dsl2Spec { upstreamTasksOf('p2 (3)') .name == ['p1 (3)'] } + + def 'should track provenance with randomSample operator'() { + when: + dsl_eval(globalConfig(), ''' + workflow { + channel.of(1..9) | p1 | randomSample(1) | p2 + } + + process p1 { + input: val(x) + output: val(y) + exec: + y = x + } + + process p2 { + input: val(x) + exec: + println x + } + ''') + + then: + upstreamTasksOf('p2 (1)') + .name.first =~ /p1 \(\d\)/ + + } } From 7ff654429e55c665ac82fa34819116f8e978e791 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sun, 9 Feb 2025 16:09:01 +0100 Subject: [PATCH 65/66] Add tap operator Signed-off-by: Paolo Di Tommaso --- .../groovy/nextflow/extension/BranchOp.groovy | 2 +- .../nextflow/extension/CollectFileOp.groovy | 2 +- .../nextflow/extension/CollectOp.groovy | 2 +- .../nextflow/extension/CombineOp.groovy | 2 +- .../groovy/nextflow/extension/ConcatOp.groovy | 2 +- .../nextflow/extension/DataflowHelper.groovy | 2 +- .../groovy/nextflow/extension/DumpOp.groovy | 2 +- .../nextflow/extension/GroupTupleOp.groovy | 2 +- .../groovy/nextflow/extension/IntoOp.groovy | 2 +- .../groovy/nextflow/extension/JoinOp.groovy | 2 +- .../groovy/nextflow/extension/LastOp.groovy | 2 +- .../groovy/nextflow/extension/MathOp.groovy | 2 +- .../groovy/nextflow/extension/MixOp.groovy | 2 +- .../nextflow/extension/MultiMapOp.groovy | 2 +- .../nextflow/extension/OperatorImpl.groovy | 6 +-- .../nextflow/extension/PublishOp.groovy | 2 +- .../nextflow/extension/RandomSampleOp.groovy | 2 +- .../groovy/nextflow/extension/SplitOp.groovy | 2 +- .../nextflow/extension/SubscribeOp.groovy | 3 +- .../groovy/nextflow/extension/TapOp.groovy | 30 +++++++---- .../groovy/nextflow/extension/ToListOp.groovy | 2 +- .../nextflow/extension/TransposeOp.groovy | 2 +- .../groovy/nextflow/extension/ViewOp.groovy | 2 +- .../nextflow/splitter/SplitterFactory.groovy | 2 +- ...pExtensionTest.groovy => TapOpTest.groovy} | 21 ++++++-- .../test/groovy/nextflow/prov/ProvTest.groovy | 52 +++++++++++++++++++ .../plugin/hello/HelloExtension.groovy | 2 +- 27 files changed, 115 insertions(+), 41 deletions(-) rename modules/nextflow/src/test/groovy/nextflow/extension/{DataflowTapExtensionTest.groovy => TapOpTest.groovy} (83%) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/BranchOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/BranchOp.groovy index 0614fe1fd6..48d66ff81b 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/BranchOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/BranchOp.groovy @@ -76,7 +76,7 @@ class BranchOp { events.put('onNext', this.&doNext) events.put('onComplete', this.&doComplete) new SubscribeOp() - .withSource(source) + .withInput(source) .withEvents(events) .apply() return this diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/CollectFileOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/CollectFileOp.groovy index efc1ac5449..93f6b9ad9e 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/CollectFileOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/CollectFileOp.groovy @@ -269,7 +269,7 @@ class CollectFileOp { @CompileStatic DataflowWriteChannel apply() { new SubscribeOp() - .withSource(channel) + .withInput(channel) .withOnNext(this.&processItem) .withOnComplete(this.&emitItems) .withContext( new ContextGrouping() ) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/CollectOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/CollectOp.groovy index f2fc3af815..ce61c2184c 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/CollectOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/CollectOp.groovy @@ -58,7 +58,7 @@ class CollectOp { final target = new DataflowVariable() new SubscribeOp() - .withSource(source) + .withInput(source) .withContext(new ContextGrouping()) .withOnNext { append(result, it) } .withOnComplete { DataflowProcessor dp -> diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/CombineOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/CombineOp.groovy index 2f3358632a..ab33f6e364 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/CombineOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/CombineOp.groovy @@ -185,7 +185,7 @@ class CombineOp { private void subscribe0(final DataflowReadChannel source, final Map events) { new SubscribeOp() - .withSource(source) + .withInput(source) .withEvents(events) .withContext(context) .apply() diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/ConcatOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/ConcatOp.groovy index a8a1159f57..4735e8b6b3 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/ConcatOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/ConcatOp.groovy @@ -58,7 +58,7 @@ class ConcatOp { final current = channels[index++] final next = index < channels.size() ? channels[index] : null new SubscribeOp() - .withSource(current) + .withInput(current) .withContext(context) .withOnNext { DataflowProcessor dp, Object it -> Op.bind(dp, result, it) } .withOnComplete { DataflowProcessor dp -> diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy index da2b84ea77..dd1ff43674 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/DataflowHelper.groovy @@ -96,7 +96,7 @@ class DataflowHelper { static final DataflowProcessor subscribeImpl(final DataflowReadChannel source, final Map events ) { new SubscribeOp() - .withSource(source) + .withInput(source) .withEvents(events) .apply() } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/DumpOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/DumpOp.groovy index 79d69ccf82..4ab0523ee6 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/DumpOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/DumpOp.groovy @@ -97,7 +97,7 @@ class DumpOp { events.onComplete = { CH.close0(target) } new SubscribeOp() - .withSource(source) + .withInput(source) .withEvents(events) .apply() diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/GroupTupleOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/GroupTupleOp.groovy index ce2218764c..dacbe39c0a 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/GroupTupleOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/GroupTupleOp.groovy @@ -249,7 +249,7 @@ class GroupTupleOp { * apply the logic to the source channel */ new SubscribeOp() - .withSource(channel) + .withInput(channel) .withContext(context) .withOnNext(this.&collectTuple) .withOnComplete(this.&finalise) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/IntoOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/IntoOp.groovy index 5afe4d1e96..b8943b4d3b 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/IntoOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/IntoOp.groovy @@ -61,7 +61,7 @@ class IntoOp { IntoOp apply() { new SubscribeOp() - .withSource(source) + .withInput(source) .withOnNext(this.&doNext) .withOnComplete(this.&doComplete) .apply() diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/JoinOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/JoinOp.groovy index 396ef3a883..2458ac59dd 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/JoinOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/JoinOp.groovy @@ -104,7 +104,7 @@ class JoinOp { private void subscribe0(DataflowReadChannel source, Map events) { new SubscribeOp() - .withSource(source) + .withInput(source) .withEvents(events) .withContext(context) .apply() diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/LastOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/LastOp.groovy index 007cf7c392..f90fd47dad 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/LastOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/LastOp.groovy @@ -54,7 +54,7 @@ class LastOp { def last = null new SubscribeOp() - .withSource(source) + .withInput(source) .withOnNext{ last = it } .withOnComplete{ DataflowProcessor dp -> Op.bind(dp, target, last) } .apply() diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/MathOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/MathOp.groovy index ec7792c59c..3283cc0838 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/MathOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/MathOp.groovy @@ -75,7 +75,7 @@ class MathOp { if( target==null ) target = new DataflowVariable() new SubscribeOp() - .withSource(source) + .withInput(source) .withContext(new ContextGrouping()) .withOnNext(aggregate.&process) .withOnComplete(this.&completion) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/MixOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/MixOp.groovy index 53b9e20a04..1fbec85049 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/MixOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/MixOp.groovy @@ -84,7 +84,7 @@ class MixOp { private void subscribe0(final DataflowReadChannel source, final Map events ) { new SubscribeOp() - .withSource(source) + .withInput(source) .withContext(context) .withEvents(events) .apply() diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/MultiMapOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/MultiMapOp.groovy index 875ca1da63..bf919cf2ed 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/MultiMapOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/MultiMapOp.groovy @@ -70,7 +70,7 @@ class MultiMapOp { MultiMapOp apply() { new SubscribeOp() - .withSource(source) + .withInput(source) .withOnNext(this.&doNext) .withOnComplete(this.&doComplete) .apply() diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy index 0211f02f10..d09394d9a0 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/OperatorImpl.groovy @@ -64,7 +64,7 @@ class OperatorImpl { */ DataflowReadChannel subscribe(final DataflowReadChannel source, final Closure closure) { new SubscribeOp() - .withSource(source) + .withInput(source) .withOnNext(closure) .apply() return source @@ -79,7 +79,7 @@ class OperatorImpl { */ DataflowReadChannel subscribe(final DataflowReadChannel source, final Map events ) { new SubscribeOp() - .withSource(source) + .withInput(source) .withEvents(events) .apply() return source @@ -744,7 +744,7 @@ class OperatorImpl { final singleton = result instanceof DataflowExpression new SubscribeOp() - .withSource(source) + .withInput(source) .withContext(new ContextRunPerThread()) .withOnNext { DataflowProcessor dp, Object it -> Op.bind(dp,result,it); empty=false } .withOnComplete { DataflowProcessor dp -> diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/PublishOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/PublishOp.groovy index f1b3a1967a..9f4ffb9174 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/PublishOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/PublishOp.groovy @@ -64,7 +64,7 @@ class PublishOp { PublishOp apply() { new SubscribeOp() - .withSource(source) + .withInput(source) .withOnNext(this.&onNext) .withOnComplete(this.&onComplete) .apply() diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/RandomSampleOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/RandomSampleOp.groovy index 3fe6f65363..367f8e9f13 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/RandomSampleOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/RandomSampleOp.groovy @@ -82,7 +82,7 @@ class RandomSampleOp { DataflowWriteChannel apply() { result = CH.create() new SubscribeOp() - .withSource(source) + .withInput(source) .withContext(context) .withOnNext(this.&sampling) .withOnComplete(this.&emit) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/SplitOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/SplitOp.groovy index 2728cdd30a..1e9622b089 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/SplitOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/SplitOp.groovy @@ -187,7 +187,7 @@ class SplitOp { @PackageScope void applySplittingOperator( DataflowReadChannel origin, DataflowWriteChannel output, AbstractSplitter splitter ) { new SubscribeOp() - .withSource(origin) + .withInput(origin) .withOnNext({ DataflowProcessor dp, entry -> splitter.processor(dp).target(entry).apply() }) .withOnComplete({ DataflowProcessor dp -> Op.bind(dp,output,Channel.STOP) }) .apply() diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/SubscribeOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/SubscribeOp.groovy index 1a91a76a30..bbba38bbcc 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/SubscribeOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/SubscribeOp.groovy @@ -46,7 +46,7 @@ class SubscribeOp { private static Session getSession() { Global.getSession() as Session } - SubscribeOp withSource(DataflowReadChannel source) { + SubscribeOp withInput(DataflowReadChannel source) { assert source!=null this.source = source return this @@ -136,6 +136,5 @@ class SubscribeOp { .withContext(context) .withCode(code) .apply() - } } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/TapOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/TapOp.groovy index e75b029e33..f7ab2ee8c2 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/TapOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/TapOp.groovy @@ -20,10 +20,11 @@ import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowWriteChannel -import groovyx.gpars.dataflow.operator.ChainWithClosure -import groovyx.gpars.dataflow.operator.CopyChannelsClosure +import groovyx.gpars.dataflow.operator.DataflowProcessor +import nextflow.Channel import nextflow.NF import nextflow.extension.op.Op + /** * Implements the {@link OperatorImpl#tap} operator * @@ -68,7 +69,7 @@ class TapOp { throw new IllegalArgumentException("Missing target channel on `tap` operator") final binding = NF.binding - names.each { item -> + for( String item : names ) { def channel = CH.createBy(source) if( binding.hasVariable(item) ) log.warn "A variable named '${item}' already exists in the script global context -- Consider renaming it " @@ -76,7 +77,6 @@ class TapOp { binding.setVariable(item, channel) outputs << channel } - } /** @@ -90,12 +90,12 @@ class TapOp { assert source != null assert target != null if( source.class != target.class ) { - throw new IllegalArgumentException("Operator `tap` source and target channel types must match -- source: ${source.class.name}, target: ${target.class.name} ") + throw new IllegalArgumentException("Operator `tap` source and target channel types must match -- source: ${source.class.name}, target: ${target.class.name} ") } this.source = source this.result = CH.createBy(source) - this.outputs = [result, target] + this.outputs = List.of(result, target) } /** @@ -108,12 +108,24 @@ class TapOp { * @return An instance of {@link TapOp} itself */ TapOp apply() { - new Op() + new SubscribeOp() .withInput(source) - .withOutputs(outputs) - .withCode(new ChainWithClosure(new CopyChannelsClosure())) + .withOnNext(this.&emit) + .withOnComplete(this.&complete) .apply() return this } + protected void emit(DataflowProcessor dp, Object it) { + for( DataflowWriteChannel ch : outputs ) { + Op.bind(dp, ch, it) + } + } + + protected void complete(DataflowProcessor dp) { + for( DataflowWriteChannel ch : outputs ) { + Op.bind(dp, ch, Channel.STOP) + } + } + } diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/ToListOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/ToListOp.groovy index 5222a8fb40..84383ef163 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/ToListOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/ToListOp.groovy @@ -53,7 +53,7 @@ class ToListOp { if( source instanceof DataflowExpression ) { final result = new ArrayList(1) new SubscribeOp() - .withSource(source) + .withInput(source) .withOnNext({ result.add(it) }) .withOnComplete({ DataflowProcessor dp -> Op.bind(dp, target, result) }) .apply() diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/TransposeOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/TransposeOp.groovy index 9d3a71812c..16dea63149 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/TransposeOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/TransposeOp.groovy @@ -65,7 +65,7 @@ class TransposeOp { DataflowWriteChannel apply() { new SubscribeOp() - .withSource(source) + .withInput(source) .withOnNext(this.&transpose) .withOnComplete(this.&done) .apply() diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/ViewOp.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/ViewOp.groovy index b100fa7680..4ddf335f1b 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/ViewOp.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/ViewOp.groovy @@ -65,7 +65,7 @@ class ViewOp { final target = CH.createBy(source); new SubscribeOp() - .withSource(source) + .withInput(source) .withOnNext { DataflowProcessor dp, Object it-> final obj = action != null ? action.call(it) : it session.printConsole(obj?.toString(), newLine) diff --git a/modules/nextflow/src/main/groovy/nextflow/splitter/SplitterFactory.groovy b/modules/nextflow/src/main/groovy/nextflow/splitter/SplitterFactory.groovy index f1bb8731e3..f479deb24d 100644 --- a/modules/nextflow/src/main/groovy/nextflow/splitter/SplitterFactory.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/splitter/SplitterFactory.groovy @@ -96,7 +96,7 @@ class SplitterFactory { strategy.options(opt) new SubscribeOp() - .withSource(source) + .withInput(source) .withOnNext({ entry -> strategy.target(entry).apply() }) .withOnComplete({ result.bind(count) }) .apply() diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/DataflowTapExtensionTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/TapOpTest.groovy similarity index 83% rename from modules/nextflow/src/test/groovy/nextflow/extension/DataflowTapExtensionTest.groovy rename to modules/nextflow/src/test/groovy/nextflow/extension/TapOpTest.groovy index 1459ab9486..4d93c83730 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/DataflowTapExtensionTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/TapOpTest.groovy @@ -21,11 +21,14 @@ import nextflow.Channel import nextflow.Session import spock.lang.Shared import spock.lang.Specification +import spock.lang.Timeout + /** * * @author Paolo Di Tommaso */ -class DataflowTapExtensionTest extends Specification { +@Timeout(10) +class TapOpTest extends Specification { @Shared Session session @@ -34,9 +37,7 @@ class DataflowTapExtensionTest extends Specification { session = new Session() } - def 'should `tap` item to a new channel' () { - when: def result = Channel.of( 4,7,9 ) .tap { first }.map { it+1 } then: @@ -51,11 +52,9 @@ class DataflowTapExtensionTest extends Specification { result.unwrap() == Channel.STOP !session.dag.isEmpty() - } def 'should `tap` item to more than one channel' () { - when: def result = Channel.of( 4,7,9 ) .tap { foo; bar }.map { it+1 } then: @@ -74,7 +73,19 @@ class DataflowTapExtensionTest extends Specification { result.unwrap() == Channel.STOP !session.dag.isEmpty() + } + def 'should `tap` item with data value' () { + when: + def result = Channel.value(1 ) .tap { first }.map { it+1 } + then: + session.binding.first.unwrap() == 1 + session.binding.first.unwrap() == 1 + and: + result.unwrap() == 2 + result.unwrap() == 2 + + !session.dag.isEmpty() } } diff --git a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy index 3897b23371..6899534c99 100644 --- a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy @@ -1408,4 +1408,56 @@ class ProvTest extends Dsl2Spec { .name.first =~ /p1 \(\d\)/ } + + def 'should track provenance with tap operator'() { + when: + dsl_eval(globalConfig(), ''' + workflow { + channel.of(1..3) | p1 + p1.out.tap { ch2 }.set{ ch3 } + p2(ch2) + p3(ch3) + } + + process p1 { + input: val(x) + output: val(y) + exec: + y = x + } + + process p2 { + input: val(x) + exec: + println x + } + + process p3 { + input: val(x) + exec: + println x + } + ''') + + then: + upstreamTasksOf('p2 (1)') + .name == ['p1 (1)'] + and: + upstreamTasksOf('p2 (2)') + .name == ['p1 (2)'] + and: + upstreamTasksOf('p2 (3)') + .name == ['p1 (3)'] + + then: + upstreamTasksOf('p3 (1)') + .name == ['p1 (1)'] + and: + upstreamTasksOf('p3 (2)') + .name == ['p1 (2)'] + and: + upstreamTasksOf('p3 (3)') + .name == ['p1 (3)'] + + } } diff --git a/modules/nf-commons/src/testFixtures/groovy/nextflow/plugin/hello/HelloExtension.groovy b/modules/nf-commons/src/testFixtures/groovy/nextflow/plugin/hello/HelloExtension.groovy index 6f35191931..049b9fa594 100644 --- a/modules/nf-commons/src/testFixtures/groovy/nextflow/plugin/hello/HelloExtension.groovy +++ b/modules/nf-commons/src/testFixtures/groovy/nextflow/plugin/hello/HelloExtension.groovy @@ -106,7 +106,7 @@ class HelloExtension extends PluginExtensionPoint { target.bind(Channel.STOP) } new SubscribeOp() - .withSource(source) + .withInput(source) .withEvents([onNext: next, onComplete: done]) .apply() return target From 06a2ac37c89dc0723bc5a7639fd6be0408867e3b Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sun, 9 Feb 2025 16:35:44 +0100 Subject: [PATCH 66/66] Add countLines and countFasta Signed-off-by: Paolo Di Tommaso --- .../groovy/nextflow/extension/op/Op.groovy | 6 +- .../nextflow/splitter/SplitterFactory.groovy | 11 +-- .../extension/CountFastaOpTest.groovy | 5 ++ .../extension/CountFastqOpTest.groovy | 11 +-- .../extension/CountLinesOpTest.groovy | 9 ++- .../test/groovy/nextflow/prov/ProvTest.groovy | 71 +++++++++++++++++++ 6 files changed, 100 insertions(+), 13 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy b/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy index b50e62f9fa..540867d233 100644 --- a/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/extension/op/Op.groovy @@ -22,8 +22,10 @@ import java.util.concurrent.ConcurrentHashMap import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import groovyx.gpars.dataflow.Dataflow +import groovyx.gpars.dataflow.DataflowChannel import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowWriteChannel +import groovyx.gpars.dataflow.expression.DataflowExpression import groovyx.gpars.dataflow.operator.DataflowEventListener import groovyx.gpars.dataflow.operator.DataflowOperator import groovyx.gpars.dataflow.operator.DataflowProcessor @@ -71,7 +73,9 @@ class Op { throw new IllegalStateException("DataflowProcessor argument cannot be null") try { if( msg instanceof PoisonPill ) { - channel.bind(msg) + final emit = channel !instanceof DataflowExpression || !(channel as DataflowChannel).isBound() + if( emit ) + channel.bind(msg) allContexts.remove(dp) } else { diff --git a/modules/nextflow/src/main/groovy/nextflow/splitter/SplitterFactory.groovy b/modules/nextflow/src/main/groovy/nextflow/splitter/SplitterFactory.groovy index f479deb24d..c0c137705f 100644 --- a/modules/nextflow/src/main/groovy/nextflow/splitter/SplitterFactory.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/splitter/SplitterFactory.groovy @@ -21,7 +21,10 @@ import groovy.util.logging.Slf4j import groovyx.gpars.dataflow.DataflowReadChannel import groovyx.gpars.dataflow.DataflowVariable import groovyx.gpars.dataflow.DataflowWriteChannel +import groovyx.gpars.dataflow.operator.DataflowProcessor import nextflow.extension.SubscribeOp +import nextflow.extension.op.Op + /** * Factory class for splitter objects * @@ -83,11 +86,10 @@ class SplitterFactory { * @return */ static DataflowWriteChannel countOverChannel(DataflowReadChannel source, SplitterStrategy splitter, Map opt ) { - // create a new DataflowChannel that will receive the splitter entries DataflowVariable result = new DataflowVariable () - def strategy = splitter as AbstractSplitter + final strategy = splitter as AbstractSplitter // set the splitter strategy options long count = 0 @@ -97,13 +99,12 @@ class SplitterFactory { new SubscribeOp() .withInput(source) - .withOnNext({ entry -> strategy.target(entry).apply() }) - .withOnComplete({ result.bind(count) }) + .withOnNext { DataflowProcessor dp, Object entry -> strategy.target(entry).apply() } + .withOnComplete { DataflowProcessor dp -> Op.bind(dp, result, count) } .apply() // return the resulting channel return result } - } diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/CountFastaOpTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/CountFastaOpTest.groovy index 2f6c9fcfd9..2644a71484 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/CountFastaOpTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/CountFastaOpTest.groovy @@ -16,6 +16,7 @@ package nextflow.extension +import nextflow.Session import spock.lang.Specification import nextflow.Channel @@ -27,6 +28,10 @@ import test.TestHelper */ class CountFastaOpTest extends Specification { + def setup() { + new Session() + } + def 'should count fasta channel' () { given: def str = ''' diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/CountFastqOpTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/CountFastqOpTest.groovy index 7d7b9fa8f5..fe675d9e30 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/CountFastqOpTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/CountFastqOpTest.groovy @@ -16,6 +16,7 @@ package nextflow.extension +import nextflow.Session import spock.lang.Specification import nextflow.Channel @@ -27,8 +28,11 @@ import test.TestHelper */ class CountFastqOpTest extends Specification { - def 'should count fastq records' () { + def setup() { + new Session() + } + def 'should count fastq records' () { given: String READS = ''' @SRR636272.19519409/1 @@ -68,8 +72,7 @@ class CountFastqOpTest extends Specification { when: def result = Channel.of( READS, READS2 ).countFastq() then: - result.val == 7 - + result.unwrap() == 7 } def 'should count fastq records from files' () { @@ -124,7 +127,7 @@ class CountFastqOpTest extends Specification { when: def result = Channel.of(file1, file2).countFastq() then: - result.val == 9 + result.unwrap() == 9 } } diff --git a/modules/nextflow/src/test/groovy/nextflow/extension/CountLinesOpTest.groovy b/modules/nextflow/src/test/groovy/nextflow/extension/CountLinesOpTest.groovy index bb0e8fa7fb..df01679d8c 100644 --- a/modules/nextflow/src/test/groovy/nextflow/extension/CountLinesOpTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/extension/CountLinesOpTest.groovy @@ -16,6 +16,7 @@ package nextflow.extension +import nextflow.Session import spock.lang.Specification import nextflow.Channel @@ -26,8 +27,11 @@ import nextflow.Channel */ class CountLinesOpTest extends Specification { - def 'should count lines' () { + def setup() { + new Session() + } + def 'should count lines' () { when: def str = ''' line 1 @@ -54,7 +58,6 @@ class CountLinesOpTest extends Specification { def result = Channel.of(str, str2, str3).countLines() then: - result.val == 11 - + result.unwrap() == 11 } } diff --git a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy index 6899534c99..99b561fe58 100644 --- a/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/prov/ProvTest.groovy @@ -1321,6 +1321,77 @@ class ProvTest extends Dsl2Spec { } + def 'should track provenance with countLines and a file'() { + given: + def FASTA = """\ + a + bb + ccc + """.stripIndent() + + when: + dsl_eval(globalConfig(), """ + workflow { + p1 | countLines | p2 + } + + process p1 { + output: path('result.txt') + exec: + task.workDir.resolve('result.txt').text = '''${FASTA}''' + } + + process p2 { + input: val(count) + exec: + println "\${task.name}: count=\$count" + } + """) + + then: + upstreamTasksOf('p2') + .name == ['p1'] + } + + def 'should track provenance with countFasta and a file'() { + given: + def FASTA = """\ + >1aboA + NLFVALYDFVASGDNTLSITKGEKLRVLGYNHNGEWCEAQTKNGQGWVPS + NYITPVN + >1ycsB + KGVIYALWDYEPQNDDELPMKEGDCMTIIHREDEDEIEWWWARLNDKEGY + VPRNLLGLYP + >1pht + GYQYRALYDYKKEREEDIDLHLGDILTVNKGSLVALGFSDGQEARPEEIG + WLNGYNETTGERGDFPGTYVE + YIGRKKISP + """.stripIndent() + + when: + dsl_eval(globalConfig(), """ + workflow { + p1 | countFasta | p2 + } + + process p1 { + output: path('result.txt') + exec: + task.workDir.resolve('result.txt').text = '''${FASTA}''' + } + + process p2 { + input: val(count) + exec: + println "\${task.name}: count=\$count" + } + """) + + then: + upstreamTasksOf('p2') + .name == ['p1'] + } + def 'should track provenance with toInteger operator'() { when: dsl_eval(globalConfig(), '''