Alternating block solve

render-ws-java-client/src/main/java/org/janelia/render/client/newsolver/DistributedAffineBlockSolver.java

@@ -9,6 +9,7 @@
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadLocalRandom;
 import java.util.stream.Collectors;
 
 import mpicbg.models.Affine2D;
@@ -67,24 +68,25 @@ public static void main(final String[] args)
                     "--project", "cut_000_to_009",
                     "--matchCollection", "c009_s310_v01_match",
                     "--stack", "c009_s310_v01_mfov_08",
-                    "--targetStack", "c009_s310_v01_align_test_overlap_shrink",
+                    "--targetStack", "c009_s310_v01_mfov_08_test",
 					"--minX", "33400",
 					"--maxX", "54600",
 					"--minY", "400",
 					"--maxY", "18700",
                     "--minZ", "424",
                     "--maxZ", "460",
 
-					"--blockSizeX", "10000",
-					"--blockSizeY", "8000",
+					"--blockSizeX", "7000",
+					"--blockSizeY", "6000",
 					// "--blockSizeZ", "100",
+					"--shiftBlocks",
 
-                    "--completeTargetStack",
+					"--completeTargetStack",
 					//"--visualizeResults",
 
 					"--maxNumMatches", "0", // no limit, default
 					"--threadsWorker", "1",
-					"--threadsGlobal", "4",
+					"--threadsGlobal", "5",
 
                     "--blockOptimizerLambdasRigid",          "1.0,1.0,0.9,0.3,0.01",
                     "--blockOptimizerLambdasTranslation",    "1.0,0.0,0.0,0.0,0.0",
@@ -103,6 +105,10 @@ public static void main(final String[] args)
         	cmdLineSetup.parse(args);
         }
 
+		run(cmdLineSetup);
+	}
+
+	public static void run(final AffineBlockSolverSetup cmdLineSetup) throws IOException, InterruptedException {
 		final RenderSetup renderSetup = RenderSetup.setupSolve(cmdLineSetup);
 
 		// Note: different setups can be used if specific things need to be done for the solve or certain blocks
@@ -200,7 +206,7 @@ private static ResultContainer<AffineModel2D> solveAndCombineBlocks(
 
 		final BlockCombiner<AffineModel2D, AffineModel2D, RigidModel2D, AffineModel2D> fusion =
 				new BlockCombiner<>(DistributedAffineBlockSolver::integrateGlobalModel,
-									DistributedAffineBlockSolver::interpolateModels);
+									DistributedAffineBlockSolver::pickRandom);
 
 		final GlobalSolver<RigidModel2D, AffineModel2D> globalSolver =
 				new GlobalSolver<>(new RigidModel2D(),
@@ -244,11 +250,36 @@ private static AffineModel2D interpolateModels(final List<AffineModel2D> models,
 		return model.createAffineModel2D();
 	}
 
+	private static AffineModel2D pickRandom(final List<AffineModel2D> models, final List<Double> weights) {
+		if (models.isEmpty() || models.size() != weights.size())
+			throw new IllegalArgumentException("models and weights must be non-empty and of the same size");
+
+		if (models.size() == 1)
+			return models.get(0);
+
+		final double randomSample = ThreadLocalRandom.current().nextDouble();
+		final int i = getRandomIndex(weights, randomSample);
+		return models.get(i);
+	}
+
+	protected static int getRandomIndex(final List<Double> weights, final double randomSample) {
+		int i;
+		double sum = 0.0;
+
+		for (i = 0; i < weights.size(); i++) {
+			sum += weights.get(i);
+			if (randomSample < sum)
+				break;
+		}
+
+		return i;
+	}
+
 	public <M extends Model<M> & Affine2D<M>, S extends Model<S> & Affine2D<S>>
 			BlockCollection<M, AffineModel2D, FIBSEMAlignmentParameters<M, S>> setupSolve(final M blockModel, final S stitchingModel)
 	{
 		// setup XY BlockFactory
-		this.blockFactory = BlockFactory.fromBlocksizes(renderSetup.getBounds(), solverSetup.blockPartition);
+		this.blockFactory = BlockFactory.fromBlockSizes(renderSetup.getBounds(), solverSetup.blockPartition);
 
 		// create all blocks
 		final BlockCollection<M, AffineModel2D, FIBSEMAlignmentParameters<M, S>> col = setupBlockCollection(this.blockFactory, blockModel, stitchingModel);
@@ -268,7 +299,7 @@ BlockCollection<M, AffineModel2D, FIBSEMAlignmentParameters<M, S>> setupBlockCol
 		} else {
 			defaultSolveParams = solverSetup.setupSolveParameters(blockModel, stitchingModel);
 		}
-		return blockFactory.defineBlockCollection(() -> defaultSolveParams);
+		return blockFactory.defineBlockCollection(() -> defaultSolveParams, solverSetup.blockPartition.shiftBlocks);
 	}
 
 	private static final Logger LOG = LoggerFactory.getLogger(DistributedAffineBlockSolver.class);

render-ws-java-client/src/main/java/org/janelia/render/client/newsolver/DistributedIntensityCorrectionSolver.java

@@ -10,6 +10,7 @@
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
+import java.util.concurrent.ThreadLocalRandom;
 
 import mpicbg.models.Affine1D;
 import mpicbg.models.AffineModel1D;
@@ -39,6 +40,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static org.janelia.render.client.newsolver.DistributedAffineBlockSolver.getRandomIndex;
+
 
 public class DistributedIntensityCorrectionSolver {
 	final IntensityCorrectionSetup solverSetup;
@@ -77,6 +80,10 @@ public static void main(final String[] args) throws IOException {
 			cmdLineSetup.parse(args);
 		}
 
+		run(cmdLineSetup);
+	}
+
+	public static void run(final IntensityCorrectionSetup cmdLineSetup) throws IOException {
 		final RenderSetup renderSetup = RenderSetup.setupSolve(cmdLineSetup);
 
 		// Note: different setups can be used if specific things need to be done for the solve or certain blocks
@@ -130,7 +137,7 @@ public ResultContainer<ArrayList<AffineModel1D>> assembleBlocks(final List<Block
 
 		final BlockCombiner<ArrayList<AffineModel1D>, ArrayList<AffineModel1D>, TranslationModel1D, ArrayList<AffineModel1D>> fusion =
 				new BlockCombiner<>(DistributedIntensityCorrectionSolver::integrateGlobalTranslation,
-									DistributedIntensityCorrectionSolver::interpolateModels);
+									DistributedIntensityCorrectionSolver::pickRandom);
 
 		final GlobalSolver<TranslationModel1D, ArrayList<AffineModel1D>> globalSolver =
 				new GlobalSolver<>(new TranslationModel1D(),
@@ -217,6 +224,19 @@ private static ArrayList<AffineModel1D> interpolateModels(final List<ArrayList<A
 		return interpolatedModels;
 	}
 
+	// TODO: remove duplication with DistributedAffineBlockSolver
+	private static ArrayList<AffineModel1D> pickRandom(final List<ArrayList<AffineModel1D>> models, final List<Double> weights) {
+		if (models.isEmpty() || models.size() != weights.size())
+			throw new IllegalArgumentException("models and weights must be non-empty and of the same size");
+
+		if (models.size() == 1)
+			return models.get(0);
+
+		final double randomSample = ThreadLocalRandom.current().nextDouble();
+		final int i = getRandomIndex(weights, randomSample);
+		return models.get(i);
+	}
+
 	private static Map<String, FilterSpec> convertCoefficientsToFilter(
 			final List<TileSpec> tiles,
 			final Map<String, ArrayList<AffineModel1D>> coefficientTiles,
@@ -267,10 +287,10 @@ private static void addFilters(final ResolvedTileSpecCollection rtsc,
 	}
 
 	public <M> BlockCollection<M, ArrayList<AffineModel1D>, FIBSEMIntensityCorrectionParameters<M>> setupSolve() {
-		this.blockFactory = BlockFactory.fromBlocksizes(renderSetup.getBounds(), solverSetup.blockPartition);
+		this.blockFactory = BlockFactory.fromBlockSizes(renderSetup.getBounds(), solverSetup.blockPartition);
 		final FIBSEMIntensityCorrectionParameters<M> defaultSolveParams = getDefaultParameters();
 		final BlockCollection<M, ArrayList<AffineModel1D>, FIBSEMIntensityCorrectionParameters<M>> col =
-				blockFactory.defineBlockCollection(() -> defaultSolveParams);
+				blockFactory.defineBlockCollection(() -> defaultSolveParams, solverSetup.blockPartition.shiftBlocks);
 
 		this.blocks = col;
 		return col;

render-ws-java-client/src/main/java/org/janelia/render/client/newsolver/blockfactories/BlockFactory.java

@@ -13,7 +13,7 @@
 
 public abstract class BlockFactory implements Serializable {
 	public abstract <M, R, P extends BlockDataSolveParameters<M, R, P>> BlockCollection<M, R, P> defineBlockCollection(
-			final ParameterProvider< M, R, P > blockSolveParameterProvider );
+			final ParameterProvider<M, R, P> blockSolveParameterProvider, final boolean shiftBlocks);
 
 	public abstract WeightFunction createWeightFunction(final BlockData<?, ?> block);
 
@@ -32,9 +32,10 @@ protected <M, R, P extends BlockDataSolveParameters<M, R, P>> BlockCollection<M,
 
 	protected abstract BlockTileBoundsFilter getBlockTileFilter();
 
-	public static BlockFactory fromBlocksizes(final Bounds range,
-											  final BlockPartitionParameters blockPartition) {
-
+	public static BlockFactory fromBlockSizes(
+			final Bounds range,
+			final BlockPartitionParameters blockPartition)
+	{
 		final int minZ = range.getMinZ().intValue();
 		final int maxZ = range.getMaxZ().intValue();
 

render-ws-java-client/src/main/java/org/janelia/render/client/newsolver/blockfactories/BlockTileBoundsFilter.java

@@ -36,6 +36,12 @@ boolean shouldBeIncluded(final Bounds tileBounds,
         return blockXYBounds.contains(tileBounds.getCenterX(), tileBounds.getCenterY());
     };
 
+    BlockTileBoundsFilter XYZ_MIDPOINT = (tileBounds, blockBounds) -> {
+        // only keep tiles where midpoint is inside block to reduce overlap
+        return BlockTileBoundsFilter.XY_MIDPOINT.shouldBeIncluded(tileBounds, blockBounds)
+                && BlockTileBoundsFilter.Z_INSIDE.shouldBeIncluded(tileBounds, blockBounds);
+    };
+
     BlockTileBoundsFilter SCALED_XY = (tileBounds, blockBounds) -> {
         // only keep tiles that touch a shrunk version of the block
         final Bounds scaledBlockBounds = blockBounds.scaled(0.75, 0.75, 1.0);

render-ws-java-client/src/main/java/org/janelia/render/client/newsolver/blockfactories/XYBlockFactory.java

@@ -57,13 +57,20 @@ public XYBlockFactory(
 
 	@Override
 	public <M, R, P extends BlockDataSolveParameters<M, R, P>> BlockCollection<M, R, P> defineBlockCollection(
-			final ParameterProvider<M, R, P> blockSolveParameterProvider)
+			final ParameterProvider<M, R, P> blockSolveParameterProvider,
+			final boolean shiftBlocks)
 	{
-		final List<Bounds> blockLayout = new BlockLayoutCreator()
-				.regularGrid(In.X, minX, maxX, blockSizeX)
-				.regularGrid(In.Y, minY, maxY, blockSizeY)
-				.singleBlock(In.Z, minZ, maxZ)
-				.create();
+		final BlockLayoutCreator creator = new BlockLayoutCreator();
+		if (shiftBlocks) {
+			creator.shiftedGrid(In.X, minX, maxX, blockSizeX);
+			creator.shiftedGrid(In.Y, minY, maxY, blockSizeY);
+		} else {
+			creator.regularGrid(In.X, minX, maxX, blockSizeX);
+			creator.regularGrid(In.Y, minY, maxY, blockSizeY);
+		}
+
+		creator.singleBlock(In.Z, minZ, maxZ);
+		final List<Bounds> blockLayout = creator.create();
 
 		// grow blocks such that they overlap
 		final List<Bounds> scaledLayout = blockLayout.stream().map(b -> b.scaled(2.0, 2.0, 1.0)).collect(Collectors.toList());
@@ -72,7 +79,7 @@ public <M, R, P extends BlockDataSolveParameters<M, R, P>> BlockCollection<M, R,
 
 	@Override
 	protected BlockTileBoundsFilter getBlockTileFilter() {
-		return BlockTileBoundsFilter.SCALED_XY;
+		return BlockTileBoundsFilter.XY_MIDPOINT;
 	}
 
 	@Override
@@ -104,7 +111,7 @@ public XYDistanceWeightFunction(final BlockData<?, ?> block, final double resolu
 			final Collection<Integer> matchedZLayers = results.getMatchedZLayers();
 			if (matchedZLayers.isEmpty()) {
 				final List<String> tileIds = results.getTileIds().stream().sorted().collect(Collectors.toList());
-				LOG.warn("XYDistanceWeightFunction ctor: block {} results with no matchedZLayers has {} tileIds: {}",
+				LOG.error("XYDistanceWeightFunction ctor: block {} results with no matchedZLayers has {} tileIds: {}",
 						 block, tileIds.size(), tileIds);
 				throw new IllegalStateException("block " + block + " has no matched z layers");
 			}

render-ws-java-client/src/main/java/org/janelia/render/client/newsolver/blockfactories/XYZBlockFactory.java

@@ -41,13 +41,20 @@ public XYZBlockFactory(
 
 	@Override
 	public <M, R, P extends BlockDataSolveParameters<M, R, P>> BlockCollection<M, R, P> defineBlockCollection(
-			final ParameterProvider<M, R, P> blockSolveParameterProvider)
+			final ParameterProvider<M, R, P> blockSolveParameterProvider,
+			final boolean shiftBlocks)
 	{
-		final List<Bounds> blockLayout = new BlockLayoutCreator()
-				.regularGrid(In.X, minX, maxX, blockSizeX)
-				.regularGrid(In.Y, minY, maxY, blockSizeY)
-				.regularGrid(In.Z, minZ, maxZ, blockSizeZ)
-				.create();
+		final BlockLayoutCreator creator = new BlockLayoutCreator();
+		if (shiftBlocks) {
+			creator.shiftedGrid(In.X, minX, maxX, blockSizeX);
+			creator.shiftedGrid(In.Y, minY, maxY, blockSizeY);
+			creator.shiftedGrid(In.Z, minZ, maxZ, blockSizeZ);
+		} else {
+			creator.regularGrid(In.X, minX, maxX, blockSizeX);
+			creator.regularGrid(In.Y, minY, maxY, blockSizeY);
+			creator.regularGrid(In.Z, minZ, maxZ, blockSizeZ);
+		}
+		final List<Bounds> blockLayout = creator.create();
 
 		// grow blocks such that they overlap
 		final List<Bounds> scaledLayout = blockLayout.stream().map(b -> b.scaled(2.0, 2.0, 2.0)).collect(Collectors.toList());
@@ -56,7 +63,7 @@ public <M, R, P extends BlockDataSolveParameters<M, R, P>> BlockCollection<M, R,
 
 	@Override
 	protected BlockTileBoundsFilter getBlockTileFilter() {
-		return BlockTileBoundsFilter.SCALED_XY_Z_INSIDE;
+		return BlockTileBoundsFilter.XYZ_MIDPOINT;
 	}
 
 	@Override

render-ws-java-client/src/main/java/org/janelia/render/client/newsolver/blockfactories/ZBlockFactory.java

@@ -34,11 +34,14 @@ public ZBlockFactory(final int minZ, final int maxZ, final int blockSize)
 
 	@Override
 	public <M, R, P extends BlockDataSolveParameters<M,R,P>> BlockCollection<M, R, P> defineBlockCollection(
-			final ParameterProvider< M, R, P > blockSolveParameterProvider )
+			final ParameterProvider<M, R, P> blockSolveParameterProvider,
+			final boolean shiftBlocks)
 	{
-		final List<Bounds> blockLayout = new BlockLayoutCreator()
-				.regularGrid(In.Z, minZ, maxZ, blockSize)
-				.create();
+		final List<Bounds> blockLayout;
+		if (shiftBlocks)
+			blockLayout = new BlockLayoutCreator().shiftedGrid(In.Z, minZ, maxZ, blockSize).create();
+		else
+			blockLayout = new BlockLayoutCreator().regularGrid(In.Z, minZ, maxZ, blockSize).create();
 
 		// grow blocks such that they overlap
 		final List<Bounds> scaledLayout = blockLayout.stream().map(b -> b.scaled(1.0, 1.0, 2.0)).collect(Collectors.toList());

render-ws-java-client/src/main/java/org/janelia/render/client/newsolver/setup/AffineBlockSolverSetup.java

@@ -10,6 +10,7 @@
 import org.janelia.alignment.spec.stack.StackId;
 import org.janelia.render.client.newsolver.blocksolveparameters.FIBSEMAlignmentParameters;
 import org.janelia.render.client.newsolver.blocksolveparameters.FIBSEMAlignmentParameters.PreAlign;
+import org.janelia.render.client.parameter.AlternatingRunParameters;
 import org.janelia.render.client.parameter.BlockOptimizerParameters;
 import org.janelia.render.client.parameter.CommandLineParameters;
 import org.janelia.render.client.parameter.MatchCollectionParameters;
@@ -52,6 +53,9 @@ public class AffineBlockSolverSetup extends CommandLineParameters
 	@ParametersDelegate
 	public BlockOptimizerParameters blockOptimizer = new BlockOptimizerParameters();
 
+	@ParametersDelegate
+	public AlternatingRunParameters alternatingRuns = new AlternatingRunParameters();
+
     @Parameter(
             names = "--stack",
             description = "Stack name",

render-ws-java-client/src/main/java/org/janelia/render/client/newsolver/setup/BlockPartitionParameters.java

@@ -12,29 +12,37 @@ public class BlockPartitionParameters implements Serializable {
 	// Initialization parameters
 	@Parameter(
 			names = "--blockSizeX",
-			description = "The x-size of the blocks which will be computed in parallel (e.g.:25000, min:1) ")
+			description = "The x-size of the blocks which will be computed in parallel (e.g.:25000, min:1)")
 	public Integer sizeX = Integer.MAX_VALUE;
 
 	@Parameter(
 			names = "--blockSizeY",
-			description = "The y-size of the blocks which will be computed in parallel (e.g.:25000, min:1) ")
+			description = "The y-size of the blocks which will be computed in parallel (e.g.:25000, min:1)")
 	public Integer sizeY = Integer.MAX_VALUE;
 
 	@Parameter(
 			names = "--blockSizeZ",
-			description = "The z-size of the blocks which will be computed in parallel (e.g.:500, min:1) ")
+			description = "The z-size of the blocks which will be computed in parallel (e.g.:500, min:1)")
 	public Integer sizeZ = Integer.MAX_VALUE;
 
+	@Parameter(
+			names = "--shiftBlocks",
+			description = "Shift blocks by half a block size in all partitioned directions",
+			arity = 0)
+	public Boolean shiftBlocks = false;
+
 	public BlockPartitionParameters() {}
 
 	public BlockPartitionParameters(
 			final Integer blockSizeX,
 			final Integer blockSizeY,
-			final Integer blockSizeZ)
+			final Integer blockSizeZ,
+			final Boolean shiftBlocks)
 	{
 		this.sizeX = (blockSizeX == null) ? Integer.MAX_VALUE : blockSizeX;
 		this.sizeY = (blockSizeY == null) ? Integer.MAX_VALUE : blockSizeY;
 		this.sizeZ = (blockSizeZ == null) ? Integer.MAX_VALUE : blockSizeZ;
+		this.shiftBlocks = (shiftBlocks != null && shiftBlocks);
 
 		ensurePositive(this.sizeX, "BlockSizeX");
 		ensurePositive(this.sizeY, "BlockSizeY");