From 56e3e67f745c99c2a272cf967cbd71a4aa2cd563 Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Wed, 10 Jan 2024 16:37:11 -0500 Subject: [PATCH] First attempt to add spark ZPositionCorrectionClient for use in pipeline. The pipeline implementation does not (yet) derive aligned stack names, so zcorr can only be run in isolation with the align stack names explicitly specified. --- .../parameter/MultiProjectParameters.java | 5 + .../parameter/StackIdWithZParameters.java | 8 +- .../client/parameter/ZSpacingParameters.java | 89 ++++ .../zspacing/ZPositionCorrectionClient.java | 108 ++--- .../pipeline/AlignmentPipelineParameters.java | 11 +- .../pipeline/AlignmentPipelineStepId.java | 4 +- .../zspacing/ZPositionCorrectionClient.java | 206 +++++++++ .../fibsem_multi_row_alignment_pipeline.json | 411 ++++++++++++++++++ 8 files changed, 757 insertions(+), 85 deletions(-) create mode 100644 render-ws-java-client/src/main/java/org/janelia/render/client/parameter/ZSpacingParameters.java create mode 100644 render-ws-spark-client/src/main/java/org/janelia/render/client/spark/zspacing/ZPositionCorrectionClient.java create mode 100644 render-ws-spark-client/src/test/resources/pipeline/fibsem_multi_row_alignment_pipeline.json diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MultiProjectParameters.java b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MultiProjectParameters.java index 619a336a7..a252ed19a 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MultiProjectParameters.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MultiProjectParameters.java @@ -90,6 +90,11 @@ public List buildListOfStackWithBatchedZ() return stackIdWithZ.buildListOfStackWithBatchedZ(this.getDataClient()); } + public List buildListOfStackWithBatchedZ(final int zValuesPerBatch) + throws IOException, IllegalArgumentException { + return stackIdWithZ.buildListOfStackWithBatchedZ(this.getDataClient(), zValuesPerBatch); + } + public List buildListOfStackWithAllZ() throws IOException, IllegalArgumentException { return stackIdWithZ.buildListOfStackWithAllZ(this.getDataClient()); diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/StackIdWithZParameters.java b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/StackIdWithZParameters.java index 03f67309f..19cf9ccc3 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/StackIdWithZParameters.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/StackIdWithZParameters.java @@ -54,7 +54,7 @@ public class StackIdWithZParameters public int zValuesPerBatch = 1; public boolean hasNoDefinedStacks() { - return (stackNames == null) || (stackNames.size() == 0); + return (stackNames == null) || stackNames.isEmpty(); } public List getStackIdList(final RenderDataClient renderDataClient) @@ -96,8 +96,8 @@ public List buildListOfStackWithAllZ(final RenderDataClient re * @return list of stack identifiers coupled with explicitZValuesPerBatch z values * that is ordered by stack and then z. */ - private List buildListOfStackWithBatchedZ(final RenderDataClient renderDataClient, - final int explicitZValuesPerBatch) + public List buildListOfStackWithBatchedZ(final RenderDataClient renderDataClient, + final int explicitZValuesPerBatch) throws IOException, IllegalArgumentException { if (explicitZValuesPerBatch < 1) { throw new IllegalArgumentException("zValuesPerBatch must be greater than zero"); @@ -121,7 +121,7 @@ private List buildListOfStackWithBatchedZ(final RenderDataClie } } - if (batchedList.size() == 0) { + if (batchedList.isEmpty()) { throw new IllegalArgumentException("no stack z-layers match parameters"); } diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/ZSpacingParameters.java b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/ZSpacingParameters.java new file mode 100644 index 000000000..015e2704b --- /dev/null +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/ZSpacingParameters.java @@ -0,0 +1,89 @@ +package org.janelia.render.client.parameter; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParametersDelegate; +import com.fasterxml.jackson.annotation.JsonIgnore; + +import java.io.FileNotFoundException; +import java.io.Serializable; + +import org.janelia.render.client.zspacing.HeadlessZPositionCorrection; +import org.janelia.render.client.zspacing.loader.ResinMaskParameters; +import org.janelia.thickness.inference.Options; + +/** + * Parameters for cross correlation based z-layer thickness correction. + * + * @author Eric Trautman + */ +public class ZSpacingParameters + implements Serializable { + + @Parameter( + names = "--scale", + description = "Scale to render each layer", + required = true) + public Double scale; + + @Parameter( + names = "--rootDirectory", + description = "Root directory for all output (e.g. /groups/flyem/data/alignment-ett/zcorr)", + required = true) + public String rootDirectory; + + @Parameter( + names = "--runName", + description = "Common run name to include in output path when running array jobs. " + + "Typically includes the timestamp when the array job was created. " + + "Omit if not running in an array job context.") + public String runName; + + @Parameter( + names = "--nLocalEstimates", + description = "Number of local estimates") + public Integer nLocalEstimates = 1; + + @ParametersDelegate + public ResinMaskParameters resin = new ResinMaskParameters(); + + @Parameter( + names = "--normalizedEdgeLayerCount", + description = "The number of layers at the beginning and end of the stack to assign correction " + + "delta values of 1.0. This hack fixes the stretched or squished corrections the " + + "solver typically produces for layers at the edges of the stack. " + + "For Z0720_07m stacks, we set this value to 30. " + + "Omit to leave the solver correction values as is.") + public Integer normalizedEdgeLayerCount; + + @Parameter( + names = "--solveExisting", + description = "Specify to load existing correlation data and solve.", + arity = 0) + public boolean solveExisting; + + @Parameter( + names = "--optionsJson", + description = "JSON file containing thickness correction options (omit to use default values)") + public String optionsJson; + + public Options inferenceOptions; + + public Options getInferenceOptions() + throws FileNotFoundException { + Options options = this.inferenceOptions; + if (options == null) { + if(optionsJson == null) { + options = HeadlessZPositionCorrection.generateDefaultFIBSEMOptions(); + } else { + options = Options.read(optionsJson); + } + } + return options; + } + + @JsonIgnore + public int getComparisonRange() + throws FileNotFoundException { + return getInferenceOptions().comparisonRange; + } +} diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/zspacing/ZPositionCorrectionClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/zspacing/ZPositionCorrectionClient.java index 1c4a73965..ff325f0a4 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/zspacing/ZPositionCorrectionClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/zspacing/ZPositionCorrectionClient.java @@ -4,7 +4,6 @@ import com.beust.jcommander.ParametersDelegate; import java.io.File; -import java.io.FileNotFoundException; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -31,8 +30,8 @@ import org.janelia.render.client.parameter.LayerBoundsParameters; import org.janelia.render.client.parameter.RenderWebServiceParameters; import org.janelia.render.client.parameter.ZRangeParameters; +import org.janelia.render.client.parameter.ZSpacingParameters; import org.janelia.render.client.zspacing.loader.RenderLayerLoader; -import org.janelia.render.client.zspacing.loader.ResinMaskParameters; import org.janelia.thickness.inference.Options; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -59,46 +58,8 @@ public static class Parameters extends CommandLineParameters { required = true) public String stack; - @Parameter( - names = "--scale", - description = "Scale to render each layer", - required = true) - public Double scale; - - @Parameter( - names = "--rootDirectory", - description = "Root directory for all output (e.g. /groups/flyem/data/alignment-ett/zcorr)", - required = true) - public String rootDirectory; - - @Parameter( - names = "--runName", - description = "Common run name to include in output path when running array jobs. " + - "Typically includes the timestamp when the array job was created. " + - "Omit if not running in an array job context.") - public String runName; - - @Parameter( - names = "--optionsJson", - description = "JSON file containing thickness correction options (omit to use default values)") - public String optionsJson; - - @Parameter( - names = "--nLocalEstimates", - description = "Number of local estimates") - public Integer nLocalEstimates = 1; - @ParametersDelegate - public ResinMaskParameters resin = new ResinMaskParameters(); - - @Parameter( - names = "--normalizedEdgeLayerCount", - description = "The number of layers at the beginning and end of the stack to assign correction " + - "delta values of 1.0. This hack fixes the stretched or squished corrections the " + - "solver typically produces for layers at the edges of the stack. " + - "For Z0720_07m stacks, we set this value to 30. " + - "Omit to leave the solver correction values as is.") - public Integer normalizedEdgeLayerCount; + public ZSpacingParameters zSpacing = new ZSpacingParameters(); @Parameter( names = "--correlationBatch", @@ -107,12 +68,6 @@ public static class Parameters extends CommandLineParameters { "(e.g. '1:20', '2:20', ..., '20:20').") public String correlationBatch; - @Parameter( - names = "--solveExisting", - description = "Specify to load existing correlation data and solve.", - arity = 0) - public boolean solveExisting; - @Parameter( names = "--poorCorrelationThreshold", description = "Generate region correlation data for layers that have correlation " + @@ -157,12 +112,6 @@ public static class Parameters extends CommandLineParameters { public Parameters() { } - public Options getInferenceOptions() - throws FileNotFoundException { - return optionsJson == null ? - HeadlessZPositionCorrection.generateDefaultFIBSEMOptions() : Options.read(optionsJson); - } - private Integer currentBatchNumber = null; private Integer totalBatchCount = null; @@ -199,14 +148,14 @@ public boolean hasBatchInfo() { public File getBaseRunDirectory() { - final String stackPath = Paths.get(rootDirectory, renderWeb.owner, renderWeb.project, stack).toString(); + final String stackPath = Paths.get(zSpacing.rootDirectory, renderWeb.owner, renderWeb.project, stack).toString(); final Path path; - if (runName == null) { + if (zSpacing.runName == null) { final SimpleDateFormat sdf = new SimpleDateFormat("'run_'yyyyMMdd_HHmmss"); path = Paths.get(stackPath, sdf.format(new Date())); } else { - path = Paths.get(stackPath, runName); + path = Paths.get(stackPath, zSpacing.runName); } return path.toFile().getAbsoluteFile(); @@ -234,7 +183,7 @@ public void runClient(final String[] args) throws Exception { client.saveRunFiles(); } - if (parameters.solveExisting) { + if (parameters.zSpacing.solveExisting) { final CrossCorrelationData ccData = client.loadCrossCorrelationDataSets(); client.estimateAndSaveZCoordinates(ccData); @@ -278,7 +227,7 @@ public void runClient(final String[] args) throws Exception { private final int firstLayerOffset; private final List stackResolutionValues; - ZPositionCorrectionClient(final Parameters parameters) + public ZPositionCorrectionClient(final Parameters parameters) throws IllegalArgumentException, IOException { this.parameters = parameters; @@ -288,7 +237,7 @@ public void runClient(final String[] args) throws Exception { final StackMetaData stackMetaData = renderDataClient.getStackMetaData(parameters.stack); this.stackResolutionValues = stackMetaData.getCurrentResolutionValues(); - this.inferenceOptions = parameters.getInferenceOptions(); + this.inferenceOptions = parameters.zSpacing.getInferenceOptions(); final List sectionDataList = renderDataClient.getStackSectionData(parameters.stack, parameters.layerRange.minZ, @@ -315,12 +264,13 @@ public void runClient(final String[] args) throws Exception { this.firstLayerOffset = allSortedZList.indexOf(this.sortedZList.get(0)); } else { this.sortedZList = allSortedZList; - this.firstLayerOffset = 0; + final List allSortedZIgnoringParameters = renderDataClient.getStackZValues(parameters.stack); + this.firstLayerOffset = allSortedZIgnoringParameters.indexOf(this.sortedZList.get(0)); } - if (parameters.runName == null) { + if (parameters.zSpacing.runName == null) { this.runDirectory = this.baseRunDirectory; - } else if (parameters.solveExisting) { + } else if (parameters.zSpacing.solveExisting) { final SimpleDateFormat sdf = new SimpleDateFormat("'solve_'yyyyMMdd_HHmmss"); this.runDirectory = new File(this.baseRunDirectory, sdf.format(new Date())); } else { // batched cross correlation run @@ -368,10 +318,10 @@ String getLayerUrlPattern(final int regionRowIndex, stackUrlString, "%s", layerBounds.getX(), layerBounds.getY(), layerBounds.getWidth(), layerBounds.getHeight(), - parameters.scale); + parameters.zSpacing.scale); } - CrossCorrelationData deriveCrossCorrelationData() + public CrossCorrelationData deriveCrossCorrelationData() throws IllegalArgumentException { LOG.info("deriveCrossCorrelationData: using comparison range: {}", inferenceOptions.comparisonRange); @@ -386,10 +336,10 @@ CrossCorrelationData deriveCrossCorrelationData() final ImageProcessorCache maskCache = new ImageProcessorCache(pixelsInLargeMask, false, false); - final RenderLayerLoader layerLoader = parameters.resin.buildLoader(layerUrlPattern, - sortedZList, - maskCache, - parameters.scale); + final RenderLayerLoader layerLoader = parameters.zSpacing.resin.buildLoader(layerUrlPattern, + sortedZList, + maskCache, + parameters.zSpacing.scale); if (parameters.debugFormat != null) { final File debugDirectory = new File(runDirectory, "debug-images"); @@ -444,10 +394,10 @@ CrossCorrelationWithNextRegionalData deriveRegionalCrossCorrelationData(final Do column, numberOfRegionRows, numberOfRegionColumns); - final RenderLayerLoader layerLoader = parameters.resin.buildLoader(layerUrlPattern, - sortedZList, - imageProcessorCache, - parameters.scale); + final RenderLayerLoader layerLoader = parameters.zSpacing.resin.buildLoader(layerUrlPattern, + sortedZList, + imageProcessorCache, + parameters.zSpacing.scale); final CrossCorrelationData crossCorrelationData = HeadlessZPositionCorrection.deriveCrossCorrelationWithCachedLoaders(layerLoader, 1, @@ -525,14 +475,14 @@ void saveRunFiles() } } - void saveCrossCorrelationData(final CrossCorrelationData ccData) + public void saveCrossCorrelationData(final CrossCorrelationData ccData) throws IOException { final String ccDataPath = new File(runDirectory, CrossCorrelationData.DEFAULT_DATA_FILE_NAME).getAbsolutePath(); FileUtil.saveJsonFile(ccDataPath, ccData); } - CrossCorrelationData loadCrossCorrelationDataSets() + public CrossCorrelationData loadCrossCorrelationDataSets() throws IllegalArgumentException, IOException { final File ccDataParent = new File(baseRunDirectory, CrossCorrelationData.DEFAULT_BATCHES_DIR_NAME); final List dataSets = @@ -542,7 +492,7 @@ CrossCorrelationData loadCrossCorrelationDataSets() return CrossCorrelationData.merge(dataSets); } - void estimateAndSaveZCoordinates(final CrossCorrelationData ccData) + public void estimateAndSaveZCoordinates(final CrossCorrelationData ccData) throws IllegalArgumentException, IOException { LOG.info("estimateAndSaveZCoordinates: using inference options: {}", inferenceOptions); @@ -553,11 +503,11 @@ void estimateAndSaveZCoordinates(final CrossCorrelationData ccData) double[] transforms = HeadlessZPositionCorrection.estimateZCoordinates(crossCorrelationMatrix, inferenceOptions, - parameters.nLocalEstimates); + parameters.zSpacing.nLocalEstimates); - if ((parameters.normalizedEdgeLayerCount != null) && - (transforms.length > (3 * parameters.normalizedEdgeLayerCount))) { - transforms = normalizeTransforms(transforms, parameters.normalizedEdgeLayerCount); + if ((parameters.zSpacing.normalizedEdgeLayerCount != null) && + (transforms.length > (3 * parameters.zSpacing.normalizedEdgeLayerCount))) { + transforms = normalizeTransforms(transforms, parameters.zSpacing.normalizedEdgeLayerCount); } final String outputFilePath = new File(runDirectory, "Zcoords.txt").getAbsolutePath(); diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineParameters.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineParameters.java index 186d246fb..99bea7b9f 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineParameters.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineParameters.java @@ -17,6 +17,7 @@ import org.janelia.render.client.parameter.MultiProjectParameters; import org.janelia.render.client.parameter.TileClusterParameters; import org.janelia.render.client.parameter.UnconnectedCrossMFOVParameters; +import org.janelia.render.client.parameter.ZSpacingParameters; import static org.janelia.alignment.json.JsonUtils.STRICT_MAPPER; @@ -37,6 +38,7 @@ public class AlignmentPipelineParameters private final TileClusterParameters tileCluster; private final MatchCopyParameters matchCopy; private final AffineBlockSolverSetup affineBlockSolverSetup; + private final ZSpacingParameters zSpacing; @SuppressWarnings("unused") public AlignmentPipelineParameters() { @@ -48,6 +50,7 @@ public AlignmentPipelineParameters() { null, null, null, + null, null); } @@ -59,7 +62,8 @@ public AlignmentPipelineParameters(final MultiProjectParameters multiProject, final UnconnectedCrossMFOVParameters unconnectedCrossMfov, final TileClusterParameters tileCluster, final MatchCopyParameters matchCopy, - final AffineBlockSolverSetup affineBlockSolverSetup) { + final AffineBlockSolverSetup affineBlockSolverSetup, + final ZSpacingParameters zSpacing) { this.multiProject = multiProject; this.pipelineSteps = pipelineSteps; this.mipmap = mipmap; @@ -69,6 +73,7 @@ public AlignmentPipelineParameters(final MultiProjectParameters multiProject, this.tileCluster = tileCluster; this.matchCopy = matchCopy; this.affineBlockSolverSetup = affineBlockSolverSetup; + this.zSpacing = zSpacing; } public MultiProjectParameters getMultiProject() { @@ -103,6 +108,10 @@ public AffineBlockSolverSetup getAffineBlockSolverSetup() { return affineBlockSolverSetup; } + public ZSpacingParameters getZSpacing() { + return zSpacing; + } + /** * @return a list of clients for each specified pipeline step. * diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineStepId.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineStepId.java index c3df716b2..028d4deb8 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineStepId.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineStepId.java @@ -9,6 +9,7 @@ import org.janelia.render.client.spark.multisem.MFOVMontageMatchPatchClient; import org.janelia.render.client.spark.multisem.UnconnectedCrossMFOVClient; import org.janelia.render.client.spark.newsolver.DistributedAffineBlockSolverClient; +import org.janelia.render.client.spark.zspacing.ZPositionCorrectionClient; /** * Identifier for a step in a spark alignment pipeline with a convenience {@link #toStepClient()} builder. @@ -23,7 +24,8 @@ public enum AlignmentPipelineStepId { FIND_UNCONNECTED_MFOVS(UnconnectedCrossMFOVClient::new), FIND_UNCONNECTED_TILES_AND_EDGES(ClusterCountClient::new), FILTER_MATCHES(CopyMatchClient::new), - ALIGN_TILES(DistributedAffineBlockSolverClient::new); + ALIGN_TILES(DistributedAffineBlockSolverClient::new), + CORRECT_Z_POSITIONS(ZPositionCorrectionClient::new); private final Supplier stepClientSupplier; diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/zspacing/ZPositionCorrectionClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/zspacing/ZPositionCorrectionClient.java new file mode 100644 index 000000000..66da944d3 --- /dev/null +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/zspacing/ZPositionCorrectionClient.java @@ -0,0 +1,206 @@ +package org.janelia.render.client.spark.zspacing; + +import com.beust.jcommander.ParametersDelegate; + +import java.io.IOException; +import java.io.Serializable; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.List; + +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.janelia.alignment.spec.stack.StackId; +import org.janelia.alignment.spec.stack.StackWithZValues; +import org.janelia.render.client.ClientRunner; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.MultiProjectParameters; +import org.janelia.render.client.parameter.ZSpacingParameters; +import org.janelia.render.client.spark.LogUtilities; +import org.janelia.render.client.spark.pipeline.AlignmentPipelineParameters; +import org.janelia.render.client.spark.pipeline.AlignmentPipelineStep; +import org.janelia.render.client.zspacing.CrossCorrelationData; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Spark client for generating z-layer cross correlation data and then using that data + * to estimate thickness for a range of z-layers in an aligned render stack. + * + * @author Eric Trautman + */ +public class ZPositionCorrectionClient + implements Serializable, AlignmentPipelineStep { + + public static class Parameters extends CommandLineParameters { + @ParametersDelegate + public MultiProjectParameters multiProject = new MultiProjectParameters(); + + @ParametersDelegate + public ZSpacingParameters zSpacing = new ZSpacingParameters(); + + public org.janelia.render.client.zspacing.ZPositionCorrectionClient buildJavaClient(final StackWithZValues stackIdWithZValues, + final int comparisonRange, + final String runName, + final boolean solveExisting) + throws IOException { + + final org.janelia.render.client.zspacing.ZPositionCorrectionClient.Parameters jClientParameters = + new org.janelia.render.client.zspacing.ZPositionCorrectionClient.Parameters(); + + jClientParameters.renderWeb.baseDataUrl = multiProject.baseDataUrl; + + final StackId stackId = stackIdWithZValues.getStackId(); + jClientParameters.renderWeb.owner = stackId.getOwner(); + jClientParameters.renderWeb.project = stackId.getProject(); + jClientParameters.stack = stackId.getStack(); + + jClientParameters.layerRange.minZ = stackIdWithZValues.getFirstZ(); + jClientParameters.layerRange.maxZ = stackIdWithZValues.getLastZ(); + + if (! solveExisting) { + // if deriving cross correlation for a batch of z layers (and not solving), + // make sure each batch overlaps enough (comparisonRange) with its prior batch + // so that nothing is missed at the batch boundaries + jClientParameters.layerRange.minZ -= comparisonRange; + } + + jClientParameters.zSpacing = zSpacing; + jClientParameters.zSpacing.runName = runName; + jClientParameters.zSpacing.solveExisting = solveExisting; + + return new org.janelia.render.client.zspacing.ZPositionCorrectionClient(jClientParameters); + } + } + + /** Run the client with command line parameters. */ + public static void main(final String[] args) { + final ClientRunner clientRunner = new ClientRunner(args) { + @Override + public void runClient(final String[] args) throws Exception { + final Parameters parameters = new Parameters(); + parameters.parse(args); + final ZPositionCorrectionClient client = new ZPositionCorrectionClient(); + client.createContextAndRun(parameters); + } + }; + clientRunner.run(); + } + + /** Empty constructor required for alignment pipeline steps. */ + public ZPositionCorrectionClient() { + } + + /** Create a spark context and run the client with the specified parameters. */ + public void createContextAndRun(final Parameters clientParameters) throws IOException { + final SparkConf conf = new SparkConf().setAppName(getClass().getSimpleName()); + try (final JavaSparkContext sparkContext = new JavaSparkContext(conf)) { + LOG.info("run: appId is {}", sparkContext.getConf().getAppId()); + deriveAndSolveCrossCorrelationData(sparkContext, clientParameters); + } + } + + /** Validates the specified pipeline parameters are sufficient. */ + @Override + public void validatePipelineParameters(final AlignmentPipelineParameters pipelineParameters) + throws IllegalArgumentException { + AlignmentPipelineParameters.validateRequiredElementExists("zSpacing", + pipelineParameters.getZSpacing()); + } + + /** Run the client as part of an alignment pipeline. */ + @Override + public void runPipelineStep(final JavaSparkContext sparkContext, + final AlignmentPipelineParameters pipelineParameters) + throws IllegalArgumentException, IOException { + + final Parameters clientParameters = new Parameters(); + clientParameters.multiProject = pipelineParameters.getMultiProject(); + clientParameters.zSpacing = pipelineParameters.getZSpacing(); + deriveAndSolveCrossCorrelationData(sparkContext, clientParameters); + } + + private void deriveAndSolveCrossCorrelationData(final JavaSparkContext sparkContext, + final Parameters clientParameters) + throws IOException { + + final int comparisonRange = clientParameters.zSpacing.getComparisonRange(); + + LOG.info("deriveAndSolveCrossCorrelationData: entry, clientParameters={}, comparisonRange={}", + clientParameters, comparisonRange); + + final MultiProjectParameters multiProjectParameters = clientParameters.multiProject; + + final List stackWithAllZValuesList = multiProjectParameters.buildListOfStackWithAllZ(); + + final long totalNumberOfZValues = stackWithAllZValuesList.stream() + .map(stackWithZ -> (long) stackWithZ.getzValues().size()) + .reduce(0L, Long::sum); + + final String runName = "run_" + new SimpleDateFormat("yyyyMMdd_HHmmss_SSS").format(new Date()) + "_z_corr"; + + deriveCrossCorrelationData(sparkContext, + clientParameters, + comparisonRange, + totalNumberOfZValues, + multiProjectParameters, + runName); + + final JavaRDD rddStackWithAllZValues = sparkContext.parallelize(stackWithAllZValuesList); + + final JavaRDD rddCompletedSolveCount = rddStackWithAllZValues.map(stackWithZ -> { + LogUtilities.setupExecutorLog4j(stackWithZ.toString()); + final org.janelia.render.client.zspacing.ZPositionCorrectionClient jClient = + clientParameters.buildJavaClient(stackWithZ, comparisonRange, runName, true); + final CrossCorrelationData ccData = jClient.loadCrossCorrelationDataSets(); + jClient.estimateAndSaveZCoordinates(ccData); + return 1; + }); + + final int completedSolveCount = rddCompletedSolveCount.collect().stream().reduce(0, Integer::sum); + + LOG.info("deriveAndSolveCrossCorrelationData: collect RDD"); + LOG.info("deriveAndSolveCrossCorrelationData: exit, solved data for {} stacks", completedSolveCount); + } + + private static void deriveCrossCorrelationData(final JavaSparkContext sparkContext, + final Parameters clientParameters, + final int comparisonRange, + final long totalNumberOfZValues, + final MultiProjectParameters multiProjectParameters, + final String runName) + throws IOException { + + final int minBatchSizeForComparisonRange = comparisonRange * 3; + final long longZValuesPerBatch = (totalNumberOfZValues / sparkContext.defaultParallelism()) + 1; + final int zValuesPerBatch = Math.max(Math.toIntExact(longZValuesPerBatch), minBatchSizeForComparisonRange); + + final List stackWithBatchedZValuesList = + multiProjectParameters.buildListOfStackWithBatchedZ(zValuesPerBatch); + + LOG.info("deriveCrossCorrelationData: created {} batches with {} layers each for {} total layers, defaultParallelism={}", + stackWithBatchedZValuesList.size(), + zValuesPerBatch, + totalNumberOfZValues, + sparkContext.defaultParallelism()); + + final JavaRDD rddStackWithBatchedZValues = sparkContext.parallelize(stackWithBatchedZValuesList); + + final JavaRDD rddCompletedBatchCount = rddStackWithBatchedZValues.map(stackWithZ -> { + LogUtilities.setupExecutorLog4j(stackWithZ.toString()); + final org.janelia.render.client.zspacing.ZPositionCorrectionClient jClient = + clientParameters.buildJavaClient(stackWithZ, comparisonRange, runName, false); + final CrossCorrelationData ccData = jClient.deriveCrossCorrelationData(); + jClient.saveCrossCorrelationData(ccData); + return 1; + }); + + final int completedBatchCount = rddCompletedBatchCount.collect().stream().reduce(0, Integer::sum); + + LOG.info("deriveCrossCorrelationData: collect RDD"); + LOG.info("deriveCrossCorrelationData: exit, generated data for {} batches", completedBatchCount); + } + + private static final Logger LOG = LoggerFactory.getLogger(ZPositionCorrectionClient.class); +} diff --git a/render-ws-spark-client/src/test/resources/pipeline/fibsem_multi_row_alignment_pipeline.json b/render-ws-spark-client/src/test/resources/pipeline/fibsem_multi_row_alignment_pipeline.json new file mode 100644 index 000000000..c55a3dced --- /dev/null +++ b/render-ws-spark-client/src/test/resources/pipeline/fibsem_multi_row_alignment_pipeline.json @@ -0,0 +1,411 @@ +{ + "multiProject": { + "baseDataUrl": "http://10.40.3.113:8080/render-ws/v1", + "owner": "cellmap", + "project": "jrc_hum_airway_14953vc", + "stackIdWithZ": { + "allStacksInProject": false, + "allStacksInAllProjects": false, + "zValuesPerBatch": 1, + "stackNames": [ + "v1_acquire" + ] + }, + "deriveMatchCollectionNamesFromProject": true + }, + "pipelineSteps": [ + "DERIVE_TILE_MATCHES", + "FIND_UNCONNECTED_TILES_AND_EDGES", + "ALIGN_TILES" + ], + "matchRunList": [ + { + "runName": "montageTopBottomRun", + "matchCommon": { + "maxPairsPerStackBatch": 30, + "featureStorage": { + "maxFeatureSourceCacheGb": 6 + }, + "maxPeakCacheGb": 2 + }, + "tilePairDerivationParameters": { + "xyNeighborFactor": 0.6, + "useRowColPositions": false, + "zNeighborDistance": 0, + "excludeCornerNeighbors": true, + "excludeCompletelyObscuredTiles": false, + "excludeSameLayerNeighbors": false, + "excludeSameSectionNeighbors": false, + "excludePairsInMatchCollection": "pairsFromPriorRuns", + "excludeSameLayerPairsWithPosition": "LEFT", + "minExistingMatchCount": 0 + }, + "matchStageParametersList": [ + { + "stageName": "montageTopBottomPass1", + "featureExtraction": { + "fdSize": 4, + "maxScale": 1.0, + "minScale": 0.25, + "steps": 5 + }, + "featureMatchDerivation": { + "matchFilter": "SINGLE_SET", + "matchFullScaleCoverageRadius": 300.0, + "matchIterations": 1000, + "matchMaxEpsilonFullScale": 5.0, + "matchMaxTrust": 4.0, + "matchMinCoveragePercentage": 0.0, + "matchMinInlierRatio": 0.0, + "matchMinNumInliers": 25, + "matchModelType": "TRANSLATION", + "matchRod": 0.92 + }, + "featureRender": { + "renderScale": 0.2, + "renderWithFilter": false, + "renderWithoutMask": false + }, + "featureRenderClip": { + "clipHeight": 300, + "clipWidth": 200 + }, + "geometricDescriptorAndMatch": { + "gdEnabled": false + } + }, + { + "stageName": "montageTopBottomPass2", + "featureExtraction": { + "fdSize": 4, + "maxScale": 1.0, + "minScale": 0.25, + "steps": 5 + }, + "featureMatchDerivation": { + "matchFilter": "SINGLE_SET", + "matchFullScaleCoverageRadius": 300.0, + "matchIterations": 1000, + "matchMaxEpsilonFullScale": 5.0, + "matchMaxTrust": 4.0, + "matchMinCoveragePercentage": 0.0, + "matchMinInlierRatio": 0.0, + "matchMinNumInliers": 12, + "matchModelType": "TRANSLATION", + "matchRod": 0.92 + }, + "featureRender": { + "renderScale": 0.4, + "renderWithFilter": false, + "renderWithoutMask": false + }, + "featureRenderClip": { + "clipHeight": 300, + "clipWidth": 200 + }, + "geometricDescriptorAndMatch": { + "gdEnabled": false + } + } + ] + }, + { + "runName": "montageLeftRightRun", + "matchCommon": { + "maxPairsPerStackBatch": 30, + "featureStorage": { + "maxFeatureSourceCacheGb": 6 + }, + "maxPeakCacheGb": 2 + }, + "tilePairDerivationParameters": { + "xyNeighborFactor": 0.6, + "useRowColPositions": false, + "zNeighborDistance": 0, + "excludeCornerNeighbors": true, + "excludeCompletelyObscuredTiles": false, + "excludeSameLayerNeighbors": false, + "excludeSameSectionNeighbors": false, + "excludePairsInMatchCollection": "pairsFromPriorRuns", + "excludeSameLayerPairsWithPosition": "TOP", + "minExistingMatchCount": 0 + }, + "matchStageParametersList": [ + { + "stageName": "montageLeftRightPass1", + "featureExtraction": { + "fdSize": 4, + "maxScale": 1.0, + "minScale": 0.25, + "steps": 5 + }, + "featureMatchDerivation": { + "matchFilter": "SINGLE_SET", + "matchFullScaleCoverageRadius": 300.0, + "matchIterations": 1000, + "matchMaxEpsilonFullScale": 5.0, + "matchMaxTrust": 4.0, + "matchMinCoveragePercentage": 0.0, + "matchMinInlierRatio": 0.0, + "matchMinNumInliers": 25, + "matchModelType": "TRANSLATION", + "matchRod": 0.92 + }, + "featureRender": { + "renderScale": 0.8, + "renderWithFilter": false, + "renderWithoutMask": false + }, + "featureRenderClip": { + "clipHeight": 300, + "clipWidth": 200 + }, + "geometricDescriptorAndMatch": { + "gdEnabled": false + } + }, + { + "stageName": "montageLeftRightPass2", + "featureExtraction": { + "fdSize": 4, + "maxScale": 1.0, + "minScale": 0.25, + "steps": 5 + }, + "featureMatchDerivation": { + "matchFilter": "SINGLE_SET", + "matchFullScaleCoverageRadius": 300.0, + "matchIterations": 1000, + "matchMaxEpsilonFullScale": 5.0, + "matchMaxTrust": 4.0, + "matchMinCoveragePercentage": 0.0, + "matchMinInlierRatio": 0.0, + "matchMinNumInliers": 6, + "matchModelType": "TRANSLATION", + "matchRod": 0.92 + }, + "featureRender": { + "renderScale": 1.0, + "renderWithFilter": false, + "renderWithoutMask": false + }, + "featureRenderClip": { + "clipHeight": 300, + "clipWidth": 200 + }, + "geometricDescriptorAndMatch": { + "gdEnabled": false + } + } + ] + }, + { + "runName": "crossRun", + "matchCommon": { + "maxPairsPerStackBatch": 60, + "featureStorage": { + "maxFeatureSourceCacheGb": 6 + }, + "maxPeakCacheGb": 2 + }, + "tilePairDerivationParameters": { + "xyNeighborFactor": 0.1, + "useRowColPositions": false, + "zNeighborDistance": 6, + "excludeCornerNeighbors": true, + "excludeCompletelyObscuredTiles": false, + "excludeSameLayerNeighbors": true, + "excludeSameSectionNeighbors": true, + "excludePairsInMatchCollection": "pairsFromPriorRuns", + "minExistingMatchCount": 0 + }, + "matchStageParametersList": [ + { + "stageName": "crossPass1", + "featureExtraction": { + "fdSize": 4, + "maxScale": 1.0, + "minScale": 0.125, + "steps": 5 + }, + "featureMatchDerivation": { + "matchFilter": "SINGLE_SET", + "matchFullScaleCoverageRadius": 300.0, + "matchIterations": 1000, + "matchMaxEpsilonFullScale": 10.0, + "matchMaxTrust": 4.0, + "matchMinCoveragePercentage": 0.0, + "matchMinInlierRatio": 0.0, + "matchMinNumInliers": 20, + "matchModelType": "RIGID", + "matchRod": 0.92 + }, + "featureRender": { + "renderScale": 0.05, + "renderWithFilter": false, + "renderWithoutMask": false + }, + "geometricDescriptorAndMatch": { + "gdEnabled": false + }, + "maxNeighborDistance": 6 + }, + { + "stageName": "crossPass2", + "featureExtraction": { + "fdSize": 4, + "maxScale": 1.0, + "minScale": 0.125, + "steps": 5 + }, + "featureMatchDerivation": { + "matchFilter": "SINGLE_SET", + "matchFullScaleCoverageRadius": 300.0, + "matchIterations": 1000, + "matchMaxEpsilonFullScale": 10.0, + "matchMaxTrust": 4.0, + "matchMinCoveragePercentage": 0.0, + "matchMinInlierRatio": 0.0, + "matchMinNumInliers": 20, + "matchModelType": "RIGID", + "matchRod": 0.92 + }, + "featureRender": { + "renderScale": 0.1, + "renderWithFilter": false, + "renderWithoutMask": false + }, + "geometricDescriptorAndMatch": { + "gdEnabled": false + }, + "maxNeighborDistance": 6 + }, + { + "stageName": "crossPass3", + "featureExtraction": { + "fdSize": 4, + "maxScale": 1.0, + "minScale": 0.125, + "steps": 5 + }, + "featureMatchDerivation": { + "matchFilter": "SINGLE_SET", + "matchFullScaleCoverageRadius": 300.0, + "matchIterations": 1000, + "matchMaxEpsilonFullScale": 10.0, + "matchMaxTrust": 4.0, + "matchMinCoveragePercentage": 0.0, + "matchMinInlierRatio": 0.0, + "matchMinNumInliers": 20, + "matchModelType": "RIGID", + "matchRod": 0.92 + }, + "featureRender": { + "renderScale": 0.2, + "renderWithFilter": false, + "renderWithoutMask": false + }, + "geometricDescriptorAndMatch": { + "gdEnabled": false + }, + "maxNeighborDistance": 3 + }, + { + "stageName": "crossPass4", + "featureExtraction": { + "fdSize": 4, + "maxScale": 1.0, + "minScale": 0.125, + "steps": 5 + }, + "featureMatchDerivation": { + "matchFilter": "SINGLE_SET", + "matchFullScaleCoverageRadius": 300.0, + "matchIterations": 1000, + "matchMaxEpsilonFullScale": 10.0, + "matchMaxTrust": 4.0, + "matchMinCoveragePercentage": 0.0, + "matchMinInlierRatio": 0.0, + "matchMinNumInliers": 10, + "matchModelType": "RIGID", + "matchRod": 0.92 + }, + "featureRender": { + "renderScale": 0.3, + "renderWithFilter": false, + "renderWithoutMask": false + }, + "geometricDescriptorAndMatch": { + "gdEnabled": false + }, + "maxNeighborDistance": 2 + } + ] + } + ], + "tileCluster": { + "maxLayersPerBatch": 1000, + "maxOverlapLayers": 10, + "includeMatchesOutsideGroup": true, + "maxLayersForUnconnectedEdge": 50 + }, + "affineBlockSolverSetup": { + "distributedSolve": { + "maxAllowedErrorGlobal": 10.0, + "maxIterationsGlobal": 10000, + "maxPlateauWidthGlobal": 50, + "deriveThreadsUsingSparkConfig": true + }, + "targetStack": { + "stackSuffix": "_align", + "completeStack": true + }, + "matches": { + "matchCollection": "jrc_hum_airway_14953vc_match" + }, + "blockPartition": { + "sizeZ": 100 + }, + "stitching": { + "minInliers": 1000000 + }, + "blockOptimizer": { + "lambdasRigid": [1.0, 1.0, 0.9, 0.3, 0.01], + "lambdasTranslation": [1.0, 0.0, 0.0, 0.0, 0.0], + "lambdasRegularization": [0.0, 0.0, 0.0, 0.0, 0.0], + "iterations": [1000, 1000, 500, 250, 250], + "maxPlateauWidth": [250, 250, 150, 100, 100] + }, + "maxNumMatches": 0, + "alternatingRuns": { + "nRuns": 4, + "keepIntermediateStacks": false + } + }, + "zSpacing": { + "scale": 0.22, + "rootDirectory": "/nrs/cellmap/data/jrc_hum-airway-14953vc/z_corr", + "runName": "run01", + "nLocalEstimates": 1, + "resin": { + "resinMaskingEnabled": true, + "resinSigma": 100, + "resinContentThreshold": 3.0, + "resinMaskIntensity": 255.0 + }, + "inferenceOptions": { + "scalingFactorRegularizerWeight" : 0.1, + "coordinateUpdateRegularizerWeight" : 0.0, + "shiftProportion" : 0.6, + "nIterations" : 100, + "comparisonRange" : 10, + "minimumSectionThickness" : 1.0E-4, + "regularizationType" : "BORDER", + "scalingFactorEstimationIterations" : 10, + "withReorder" : false, + "forceMonotonicity" : false, + "estimateWindowRadius" : -1, + "minimumCorrelationValue" : 0.0 + } + } +} \ No newline at end of file