Skip to content

Commit

Permalink
First attempt to add spark ZPositionCorrectionClient for use in pipel…
Browse files Browse the repository at this point in the history
…ine. The pipeline implementation does not (yet) derive aligned stack names, so zcorr can only be run in isolation with the align stack names explicitly specified.
  • Loading branch information
trautmane committed Jan 10, 2024
1 parent 19e26ab commit 56e3e67
Show file tree
Hide file tree
Showing 8 changed files with 757 additions and 85 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,11 @@ public List<StackWithZValues> buildListOfStackWithBatchedZ()
return stackIdWithZ.buildListOfStackWithBatchedZ(this.getDataClient());
}

public List<StackWithZValues> buildListOfStackWithBatchedZ(final int zValuesPerBatch)
throws IOException, IllegalArgumentException {
return stackIdWithZ.buildListOfStackWithBatchedZ(this.getDataClient(), zValuesPerBatch);
}

public List<StackWithZValues> buildListOfStackWithAllZ()
throws IOException, IllegalArgumentException {
return stackIdWithZ.buildListOfStackWithAllZ(this.getDataClient());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ public class StackIdWithZParameters
public int zValuesPerBatch = 1;

public boolean hasNoDefinedStacks() {
return (stackNames == null) || (stackNames.size() == 0);
return (stackNames == null) || stackNames.isEmpty();
}

public List<StackId> getStackIdList(final RenderDataClient renderDataClient)
Expand Down Expand Up @@ -96,8 +96,8 @@ public List<StackWithZValues> buildListOfStackWithAllZ(final RenderDataClient re
* @return list of stack identifiers coupled with explicitZValuesPerBatch z values
* that is ordered by stack and then z.
*/
private List<StackWithZValues> buildListOfStackWithBatchedZ(final RenderDataClient renderDataClient,
final int explicitZValuesPerBatch)
public List<StackWithZValues> buildListOfStackWithBatchedZ(final RenderDataClient renderDataClient,
final int explicitZValuesPerBatch)
throws IOException, IllegalArgumentException {
if (explicitZValuesPerBatch < 1) {
throw new IllegalArgumentException("zValuesPerBatch must be greater than zero");
Expand All @@ -121,7 +121,7 @@ private List<StackWithZValues> buildListOfStackWithBatchedZ(final RenderDataClie
}
}

if (batchedList.size() == 0) {
if (batchedList.isEmpty()) {
throw new IllegalArgumentException("no stack z-layers match parameters");
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
package org.janelia.render.client.parameter;

import com.beust.jcommander.Parameter;
import com.beust.jcommander.ParametersDelegate;
import com.fasterxml.jackson.annotation.JsonIgnore;

import java.io.FileNotFoundException;
import java.io.Serializable;

import org.janelia.render.client.zspacing.HeadlessZPositionCorrection;
import org.janelia.render.client.zspacing.loader.ResinMaskParameters;
import org.janelia.thickness.inference.Options;

/**
* Parameters for cross correlation based z-layer thickness correction.
*
* @author Eric Trautman
*/
public class ZSpacingParameters
implements Serializable {

@Parameter(
names = "--scale",
description = "Scale to render each layer",
required = true)
public Double scale;

@Parameter(
names = "--rootDirectory",
description = "Root directory for all output (e.g. /groups/flyem/data/alignment-ett/zcorr)",
required = true)
public String rootDirectory;

@Parameter(
names = "--runName",
description = "Common run name to include in output path when running array jobs. " +
"Typically includes the timestamp when the array job was created. " +
"Omit if not running in an array job context.")
public String runName;

@Parameter(
names = "--nLocalEstimates",
description = "Number of local estimates")
public Integer nLocalEstimates = 1;

@ParametersDelegate
public ResinMaskParameters resin = new ResinMaskParameters();

@Parameter(
names = "--normalizedEdgeLayerCount",
description = "The number of layers at the beginning and end of the stack to assign correction " +
"delta values of 1.0. This hack fixes the stretched or squished corrections the " +
"solver typically produces for layers at the edges of the stack. " +
"For Z0720_07m stacks, we set this value to 30. " +
"Omit to leave the solver correction values as is.")
public Integer normalizedEdgeLayerCount;

@Parameter(
names = "--solveExisting",
description = "Specify to load existing correlation data and solve.",
arity = 0)
public boolean solveExisting;

@Parameter(
names = "--optionsJson",
description = "JSON file containing thickness correction options (omit to use default values)")
public String optionsJson;

public Options inferenceOptions;

public Options getInferenceOptions()
throws FileNotFoundException {
Options options = this.inferenceOptions;
if (options == null) {
if(optionsJson == null) {
options = HeadlessZPositionCorrection.generateDefaultFIBSEMOptions();
} else {
options = Options.read(optionsJson);
}
}
return options;
}

@JsonIgnore
public int getComparisonRange()
throws FileNotFoundException {
return getInferenceOptions().comparisonRange;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
import com.beust.jcommander.ParametersDelegate;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
Expand All @@ -31,8 +30,8 @@
import org.janelia.render.client.parameter.LayerBoundsParameters;
import org.janelia.render.client.parameter.RenderWebServiceParameters;
import org.janelia.render.client.parameter.ZRangeParameters;
import org.janelia.render.client.parameter.ZSpacingParameters;
import org.janelia.render.client.zspacing.loader.RenderLayerLoader;
import org.janelia.render.client.zspacing.loader.ResinMaskParameters;
import org.janelia.thickness.inference.Options;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand All @@ -59,46 +58,8 @@ public static class Parameters extends CommandLineParameters {
required = true)
public String stack;

@Parameter(
names = "--scale",
description = "Scale to render each layer",
required = true)
public Double scale;

@Parameter(
names = "--rootDirectory",
description = "Root directory for all output (e.g. /groups/flyem/data/alignment-ett/zcorr)",
required = true)
public String rootDirectory;

@Parameter(
names = "--runName",
description = "Common run name to include in output path when running array jobs. " +
"Typically includes the timestamp when the array job was created. " +
"Omit if not running in an array job context.")
public String runName;

@Parameter(
names = "--optionsJson",
description = "JSON file containing thickness correction options (omit to use default values)")
public String optionsJson;

@Parameter(
names = "--nLocalEstimates",
description = "Number of local estimates")
public Integer nLocalEstimates = 1;

@ParametersDelegate
public ResinMaskParameters resin = new ResinMaskParameters();

@Parameter(
names = "--normalizedEdgeLayerCount",
description = "The number of layers at the beginning and end of the stack to assign correction " +
"delta values of 1.0. This hack fixes the stretched or squished corrections the " +
"solver typically produces for layers at the edges of the stack. " +
"For Z0720_07m stacks, we set this value to 30. " +
"Omit to leave the solver correction values as is.")
public Integer normalizedEdgeLayerCount;
public ZSpacingParameters zSpacing = new ZSpacingParameters();

@Parameter(
names = "--correlationBatch",
Expand All @@ -107,12 +68,6 @@ public static class Parameters extends CommandLineParameters {
"(e.g. '1:20', '2:20', ..., '20:20').")
public String correlationBatch;

@Parameter(
names = "--solveExisting",
description = "Specify to load existing correlation data and solve.",
arity = 0)
public boolean solveExisting;

@Parameter(
names = "--poorCorrelationThreshold",
description = "Generate region correlation data for layers that have correlation " +
Expand Down Expand Up @@ -157,12 +112,6 @@ public static class Parameters extends CommandLineParameters {
public Parameters() {
}

public Options getInferenceOptions()
throws FileNotFoundException {
return optionsJson == null ?
HeadlessZPositionCorrection.generateDefaultFIBSEMOptions() : Options.read(optionsJson);
}

private Integer currentBatchNumber = null;
private Integer totalBatchCount = null;

Expand Down Expand Up @@ -199,14 +148,14 @@ public boolean hasBatchInfo() {

public File getBaseRunDirectory() {

final String stackPath = Paths.get(rootDirectory, renderWeb.owner, renderWeb.project, stack).toString();
final String stackPath = Paths.get(zSpacing.rootDirectory, renderWeb.owner, renderWeb.project, stack).toString();

final Path path;
if (runName == null) {
if (zSpacing.runName == null) {
final SimpleDateFormat sdf = new SimpleDateFormat("'run_'yyyyMMdd_HHmmss");
path = Paths.get(stackPath, sdf.format(new Date()));
} else {
path = Paths.get(stackPath, runName);
path = Paths.get(stackPath, zSpacing.runName);
}

return path.toFile().getAbsoluteFile();
Expand Down Expand Up @@ -234,7 +183,7 @@ public void runClient(final String[] args) throws Exception {
client.saveRunFiles();
}

if (parameters.solveExisting) {
if (parameters.zSpacing.solveExisting) {

final CrossCorrelationData ccData = client.loadCrossCorrelationDataSets();
client.estimateAndSaveZCoordinates(ccData);
Expand Down Expand Up @@ -278,7 +227,7 @@ public void runClient(final String[] args) throws Exception {
private final int firstLayerOffset;
private final List<Double> stackResolutionValues;

ZPositionCorrectionClient(final Parameters parameters)
public ZPositionCorrectionClient(final Parameters parameters)
throws IllegalArgumentException, IOException {

this.parameters = parameters;
Expand All @@ -288,7 +237,7 @@ public void runClient(final String[] args) throws Exception {
final StackMetaData stackMetaData = renderDataClient.getStackMetaData(parameters.stack);
this.stackResolutionValues = stackMetaData.getCurrentResolutionValues();

this.inferenceOptions = parameters.getInferenceOptions();
this.inferenceOptions = parameters.zSpacing.getInferenceOptions();

final List<SectionData> sectionDataList = renderDataClient.getStackSectionData(parameters.stack,
parameters.layerRange.minZ,
Expand All @@ -315,12 +264,13 @@ public void runClient(final String[] args) throws Exception {
this.firstLayerOffset = allSortedZList.indexOf(this.sortedZList.get(0));
} else {
this.sortedZList = allSortedZList;
this.firstLayerOffset = 0;
final List<Double> allSortedZIgnoringParameters = renderDataClient.getStackZValues(parameters.stack);
this.firstLayerOffset = allSortedZIgnoringParameters.indexOf(this.sortedZList.get(0));
}

if (parameters.runName == null) {
if (parameters.zSpacing.runName == null) {
this.runDirectory = this.baseRunDirectory;
} else if (parameters.solveExisting) {
} else if (parameters.zSpacing.solveExisting) {
final SimpleDateFormat sdf = new SimpleDateFormat("'solve_'yyyyMMdd_HHmmss");
this.runDirectory = new File(this.baseRunDirectory, sdf.format(new Date()));
} else { // batched cross correlation run
Expand Down Expand Up @@ -368,10 +318,10 @@ String getLayerUrlPattern(final int regionRowIndex,
stackUrlString, "%s",
layerBounds.getX(), layerBounds.getY(),
layerBounds.getWidth(), layerBounds.getHeight(),
parameters.scale);
parameters.zSpacing.scale);
}

CrossCorrelationData deriveCrossCorrelationData()
public CrossCorrelationData deriveCrossCorrelationData()
throws IllegalArgumentException {

LOG.info("deriveCrossCorrelationData: using comparison range: {}", inferenceOptions.comparisonRange);
Expand All @@ -386,10 +336,10 @@ CrossCorrelationData deriveCrossCorrelationData()
final ImageProcessorCache maskCache = new ImageProcessorCache(pixelsInLargeMask,
false,
false);
final RenderLayerLoader layerLoader = parameters.resin.buildLoader(layerUrlPattern,
sortedZList,
maskCache,
parameters.scale);
final RenderLayerLoader layerLoader = parameters.zSpacing.resin.buildLoader(layerUrlPattern,
sortedZList,
maskCache,
parameters.zSpacing.scale);

if (parameters.debugFormat != null) {
final File debugDirectory = new File(runDirectory, "debug-images");
Expand Down Expand Up @@ -444,10 +394,10 @@ CrossCorrelationWithNextRegionalData deriveRegionalCrossCorrelationData(final Do
column,
numberOfRegionRows,
numberOfRegionColumns);
final RenderLayerLoader layerLoader = parameters.resin.buildLoader(layerUrlPattern,
sortedZList,
imageProcessorCache,
parameters.scale);
final RenderLayerLoader layerLoader = parameters.zSpacing.resin.buildLoader(layerUrlPattern,
sortedZList,
imageProcessorCache,
parameters.zSpacing.scale);
final CrossCorrelationData crossCorrelationData =
HeadlessZPositionCorrection.deriveCrossCorrelationWithCachedLoaders(layerLoader,
1,
Expand Down Expand Up @@ -525,14 +475,14 @@ void saveRunFiles()
}
}

void saveCrossCorrelationData(final CrossCorrelationData ccData)
public void saveCrossCorrelationData(final CrossCorrelationData ccData)
throws IOException {
final String ccDataPath = new File(runDirectory,
CrossCorrelationData.DEFAULT_DATA_FILE_NAME).getAbsolutePath();
FileUtil.saveJsonFile(ccDataPath, ccData);
}

CrossCorrelationData loadCrossCorrelationDataSets()
public CrossCorrelationData loadCrossCorrelationDataSets()
throws IllegalArgumentException, IOException {
final File ccDataParent = new File(baseRunDirectory, CrossCorrelationData.DEFAULT_BATCHES_DIR_NAME);
final List<CrossCorrelationData> dataSets =
Expand All @@ -542,7 +492,7 @@ CrossCorrelationData loadCrossCorrelationDataSets()
return CrossCorrelationData.merge(dataSets);
}

void estimateAndSaveZCoordinates(final CrossCorrelationData ccData)
public void estimateAndSaveZCoordinates(final CrossCorrelationData ccData)
throws IllegalArgumentException, IOException {

LOG.info("estimateAndSaveZCoordinates: using inference options: {}", inferenceOptions);
Expand All @@ -553,11 +503,11 @@ void estimateAndSaveZCoordinates(final CrossCorrelationData ccData)

double[] transforms = HeadlessZPositionCorrection.estimateZCoordinates(crossCorrelationMatrix,
inferenceOptions,
parameters.nLocalEstimates);
parameters.zSpacing.nLocalEstimates);

if ((parameters.normalizedEdgeLayerCount != null) &&
(transforms.length > (3 * parameters.normalizedEdgeLayerCount))) {
transforms = normalizeTransforms(transforms, parameters.normalizedEdgeLayerCount);
if ((parameters.zSpacing.normalizedEdgeLayerCount != null) &&
(transforms.length > (3 * parameters.zSpacing.normalizedEdgeLayerCount))) {
transforms = normalizeTransforms(transforms, parameters.zSpacing.normalizedEdgeLayerCount);
}

final String outputFilePath = new File(runDirectory, "Zcoords.txt").getAbsolutePath();
Expand Down
Loading

0 comments on commit 56e3e67

Please sign in to comment.