Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

newsolver_wafer_60_match_patch_2 #204

Merged
merged 4 commits into from
Feb 27, 2025
+147 −38
Merged

newsolver_wafer_60_match_patch_2 #204

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
171a72c
add trimMfovsWithNoConnectedTiles parameter for use by spark MFOVMont…
trautmane Feb 27, 2025
2f8b369
add support for creating and completing trimmed stacks to spark MFOVM…
trautmane Feb 27, 2025
ea172ec
Add support for new patch.trimMfovsWithNoConnectedTiles parameter. I…
trautmane Feb 27, 2025
6bee793
update ClusterCountClientTest main parameters to check w60_s360_r00_d…
trautmane Feb 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
110 changes: 76 additions & 34 deletions ...-client/src/main/java/org/janelia/render/client/multisem/MFOVMontageMatchPatchClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,12 @@ public int deriveAndSaveMatchesForUnconnectedPairsInStack(final RenderDataClient

for (final Double z : zToSectionIdsMap.keySet().stream().sorted().collect(Collectors.toList())) {
final Set<String> sectionIds = zToSectionIdsMap.get(z);
updatePositionPairDataForZ(stack, z, sectionIds, positionToPairs, renderDataClient, matchClient);
updatePositionPairDataForZ(stack,
z,
sectionIds,
positionToPairs,
renderDataClient,
matchClient);
}

final int totalNumberOfPositions = positionToPairs.size();
Expand Down Expand Up @@ -245,64 +250,101 @@ public void updatePositionPairDataForZ(final String stack,
false,
false);

// add all MFOV tile pairs to unconnected set
// add all MFOV tile pairs to unconnected set to start
final Set<OrderedCanvasIdPair> unconnectedPairsForMFOV = new HashSet<>(potentialPairsForZ.size());
final Set<String> mfovTileIds = new HashSet<>();
for (final OrderedCanvasIdPair pair : potentialPairsForZ) {
final String pMagcMfovSfov = MultiSemUtilities.getMagcMfovSfovForTileId(pair.getP().getId());
final String qMagcMfovSfov = MultiSemUtilities.getMagcMfovSfovForTileId(pair.getQ().getId());

final CanvasId p = pair.getP();
final String pId = p.getId();
final String pMagcMfovSfov = MultiSemUtilities.getMagcMfovSfovForTileId(pId);

final CanvasId q = pair.getQ();
final String qId = q.getId();
final String qMagcMfovSfov = MultiSemUtilities.getMagcMfovSfovForTileId(qId);

if (pMagcMfovSfov.startsWith(patch.pMagcMfovSfovPrefix) &&
qMagcMfovSfov.startsWith(patch.qMagcMfovSfovPrefix)) {

mfovTileIds.add(pId);
mfovTileIds.add(qId);

// remove relative position info from tree search to simplify existence check later
final OrderedCanvasIdPair pairWithoutRelative =
new OrderedCanvasIdPair(pair.getP().withoutRelativePosition(),
pair.getQ().withoutRelativePosition(),
new OrderedCanvasIdPair(p.withoutRelativePosition(),
q.withoutRelativePosition(),
0.0);
final MFOVPositionPair positionPair = new MFOVPositionPair(pairWithoutRelative);

final MFOVPositionPairMatchData positionPairMatchData =
positionToPairs.computeIfAbsent(positionPair,
d -> new MFOVPositionPairMatchData(positionPair));

positionPairMatchData.addPair(pairWithoutRelative,
resolvedTiles.getTileSpec(pair.getP().getId()),
resolvedTiles.getTileSpec(pair.getQ().getId()));
resolvedTiles.getTileSpec(pId),
resolvedTiles.getTileSpec(qId));

unconnectedPairsForMFOV.add(pairWithoutRelative);
}
}

LOG.info("updatePositionPairDataForZ: found {} pairs within mFOV {} in z {}",
unconnectedPairsForMFOV.size(), patch.multiFieldOfViewId, z);
LOG.info("updatePositionPairDataForZ: found {} tiles and {} pairs within mFOV {} in z {}",
mfovTileIds.size(), unconnectedPairsForMFOV.size(), patch.multiFieldOfViewId, z);

// query web service to find connected tile pairs and remove them from unconnected set
final Map<String, OrderedCanvasIdPair> sameLayerPairsFromOtherMFOVs = new HashMap<>();
int numberOfConnectedPairsForMFOV = 0;
if (! unconnectedPairsForMFOV.isEmpty()) {


for (final String groupId : sectionIds) {
for (final CanvasMatches canvasMatches : matchClient.getMatchesWithinGroup(groupId,
true)) {
final String pId = canvasMatches.getpId();
final String qId = canvasMatches.getqId();
final String pMagcMfovSfov = MultiSemUtilities.getMagcMfovSfovForTileId(pId);
final String qMagcMfovSfov = MultiSemUtilities.getMagcMfovSfovForTileId(qId);
final OrderedCanvasIdPair pair = new OrderedCanvasIdPair(new CanvasId(groupId, pId),
new CanvasId(groupId, qId),
0.0);
if (pMagcMfovSfov.startsWith(patch.pMagcMfovSfovPrefix) &&
qMagcMfovSfov.startsWith(patch.qMagcMfovSfovPrefix)) {
if (! unconnectedPairsForMFOV.remove(pair)) {
LOG.warn("updatePositionPairDataForZ: failed to locate existing pair {} in potential set",
pair);
}
} else {
// TODO: as coded here, same layer pair will often come from last MFOV (19) - does it matter?
final String sfovIndexPairName = MultiSemUtilities.getSFOVIndexPairName(canvasMatches.getpGroupId(),
pId,
qId);
sameLayerPairsFromOtherMFOVs.put(sfovIndexPairName, pair);
}
}
for (final CanvasMatches canvasMatches : matchClient.getMatchesWithinGroup(groupId,
true)) {
final String pId = canvasMatches.getpId();
final String qId = canvasMatches.getqId();
final String pMagcMfovSfov = MultiSemUtilities.getMagcMfovSfovForTileId(pId);
final String qMagcMfovSfov = MultiSemUtilities.getMagcMfovSfovForTileId(qId);
final OrderedCanvasIdPair pair = new OrderedCanvasIdPair(new CanvasId(groupId, pId),
new CanvasId(groupId, qId),
0.0);
if (pMagcMfovSfov.startsWith(patch.pMagcMfovSfovPrefix) &&
qMagcMfovSfov.startsWith(patch.qMagcMfovSfovPrefix)) {
if (unconnectedPairsForMFOV.remove(pair)) {
numberOfConnectedPairsForMFOV++;
} else {
LOG.warn("updatePositionPairDataForZ: failed to locate existing pair {} in potential set",
pair);
}
} else {
// note that same layer pair will often come from last MFOV (e.g. MFOV 19 for wafer 53)
final String sfovIndexPairName = MultiSemUtilities.getSFOVIndexPairName(canvasMatches.getpGroupId(),
pId,
qId);
sameLayerPairsFromOtherMFOVs.put(sfovIndexPairName, pair);
}
}
}
}

final String trimStack = patch.getTrimStackName(stack);

LOG.info("updatePositionPairDataForZ: found {} connected pairs and {} unconnected pairs within mFOV {} in z {}, trimStack={}",
numberOfConnectedPairsForMFOV, unconnectedPairsForMFOV.size(), patch.multiFieldOfViewId, z, trimStack);

if (trimStack != null) {

if (numberOfConnectedPairsForMFOV == 0) {
LOG.info("updatePositionPairDataForZ: all pairs in mFOV {} of z {} are unconnected, excluding this MFOV from the {} stack and skipping patch",
patch.multiFieldOfViewId, z, trimStack);
unconnectedPairsForMFOV.clear();
} else {
LOG.info("updatePositionPairDataForZ: since {} connected pairs were found for mFOV {} in z {}, copying all {} tiles to the {} stack",
numberOfConnectedPairsForMFOV, patch.multiFieldOfViewId, z, mfovTileIds.size(), trimStack);
final ResolvedTileSpecCollection mfovResolvedTiles = resolvedTiles.copyAndRetainTileSpecs(mfovTileIds);
renderDataClient.saveResolvedTiles(mfovResolvedTiles, trimStack, z);
}

}

for (final OrderedCanvasIdPair unconnectedPair : unconnectedPairsForMFOV) {

final MFOVPositionPair positionPair = new MFOVPositionPair(unconnectedPair);
Expand Down
13 changes: 13 additions & 0 deletions ...nt/src/main/java/org/janelia/render/client/parameter/MFOVMontageMatchPatchParameters.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,14 @@ public class MFOVMontageMatchPatchParameters
description = "Number of MFOVs to process in each batch")
public int numberOfMFOVsPerBatch = 1;

@Parameter(
names = "--trimMfovsWithNoConnectedTiles",
description = "If specified, create a 'trim' copy of the source stack that excludes any unconnected " +
"tiles that exist in a z-layer MFOV in which all of its tiles are completely unconnected. " +
"Note that this option is currently only supported for Spark jobs.",
arity = 0)
public boolean trimMfovsWithNoConnectedTiles = false;

public MFOVMontageMatchPatchParameters() {
}

Expand All @@ -115,6 +123,7 @@ public MFOVMontageMatchPatchParameters withMultiFieldOfViewId(final String multi
clonedParameters.pTileId = this.pTileId;
clonedParameters.qTileId = this.qTileId;
clonedParameters.matchStorageFile = this.matchStorageFile;
clonedParameters.trimMfovsWithNoConnectedTiles = this.trimMfovsWithNoConnectedTiles;
clonedParameters.validateAndSetupDerivedValues(); // make sure values derived from multiFieldOfViewId are rebuilt
return clonedParameters;
}
Expand Down Expand Up @@ -166,6 +175,10 @@ public List<List<StackMFOVWithZValues>> bundleMFOVs(final List<StackMFOVWithZVal
return bundles;
}

public String getTrimStackName(final String sourceStackName) {
return trimMfovsWithNoConnectedTiles ? sourceStackName + "_trim" : null;
}

public static MFOVMontageMatchPatchParameters fromJson(final Reader json) {
return JSON_HELPER.fromJson(json);
}
Expand Down
8 changes: 4 additions & 4 deletions render-ws-java-client/src/test/java/org/janelia/render/client/ClusterCountClientTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,10 @@ public static void main(final String[] args) {

final String[] effectiveArgs = (args != null) && (args.length > 0) ? args : new String[]{
"--baseDataUrl", "http://renderer-dev:8080/render-ws/v1",
"--owner", "trautmane",
"--project", "w60_serial_290_to_299",
"--stack", "w60_s296_r00_d00_google_cluster0",
"--matchCollection", "w60_s296_r00_d00_google_cluster0_match",
"--owner", "hess_wafers_60_61",
"--project", "w60_serial_360_to_369",
"--stack", "w60_s360_r00_d20_gc_trim",
"--matchCollection", "w60_s360_r00_d20_gc_match",
"--maxSmallClusterSize", "0",
"--includeMatchesOutsideGroup",
"--maxLayersPerBatch", "1000",
Expand Down
54 changes: 54 additions & 0 deletions ...t/src/main/java/org/janelia/render/client/spark/multisem/MFOVMontageMatchPatchClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;

import org.apache.spark.SparkConf;
Expand All @@ -14,6 +15,8 @@
import org.janelia.alignment.match.MatchCollectionId;
import org.janelia.alignment.multisem.StackMFOVWithZValues;
import org.janelia.alignment.spec.stack.StackId;
import org.janelia.alignment.spec.stack.StackMetaData;
import org.janelia.alignment.spec.stack.StackWithZValues;
import org.janelia.render.client.ClientRunner;
import org.janelia.render.client.RenderDataClient;
import org.janelia.render.client.parameter.CommandLineParameters;
Expand Down Expand Up @@ -61,6 +64,8 @@ public void runClient(final String[] args) throws Exception {
clientRunner.run();
}

private final List<StackId> trimStackIdList = new ArrayList<>();

/** Empty constructor required for alignment pipeline steps. */
public MFOVMontageMatchPatchClient() {
}
Expand Down Expand Up @@ -135,6 +140,10 @@ private void patchPairsForPass(final JavaSparkContext sparkContext,
final List<StackMFOVWithZValues> stackMFOVWithZValuesList =
multiProjectParameters.buildListOfStackMFOVWithAllZ(patchParameters.getMultiFieldOfViewId());

if (patchParameters.trimMfovsWithNoConnectedTiles) {
createTrimStacks(multiProjectParameters, patchParameters);
}

final List<List<StackMFOVWithZValues>> bundledMFOVList = patchParameters.bundleMFOVs(stackMFOVWithZValuesList);

LOG.info("patchPairsForPass: {}, distributing tasks for {} bundles of {} MFOVs",
Expand Down Expand Up @@ -181,9 +190,54 @@ private void patchPairsForPass(final JavaSparkContext sparkContext,
.mapToLong(Integer::longValue)
.reduce(0, Long::sum);

if (patchParameters.trimMfovsWithNoConnectedTiles) {
completeTrimStacks(multiProjectParameters);
}

LOG.info("patchPairsForPass: {}, exit, derived matches for {} tile pairs",
passName, numberOfDerivedMatchPairs);
}

private void createTrimStacks(final MultiProjectParameters multiProject,
final MFOVMontageMatchPatchParameters patchParameters)
throws IOException {

LOG.info("createTrimStacks: entry");

final List<StackWithZValues> stackWithAllZValuesList = multiProject.buildListOfStackWithAllZ();
final RenderDataClient defaultDataClient = multiProject.getDataClient();

for (final StackWithZValues stackWithZValues : stackWithAllZValuesList) {

final StackId stackId = stackWithZValues.getStackId();
final RenderDataClient stackDataClient = defaultDataClient.buildClient(stackId.getOwner(),
stackId.getProject());
final StackMetaData stackMetaData = stackDataClient.getStackMetaData(stackId.getStack());
final String trimStackName = patchParameters.getTrimStackName(stackId.getStack());

stackDataClient.setupDerivedStack(stackMetaData, trimStackName);

trimStackIdList.add(new StackId(stackId.getOwner(), stackId.getProject(), trimStackName));
}

LOG.info("createTrimStacks: exit");
}

private void completeTrimStacks(final MultiProjectParameters multiProject)
throws IOException {

LOG.info("completeTrimStacks: entry");

final RenderDataClient defaultDataClient = multiProject.getDataClient();

for (final StackId trimStackId : trimStackIdList) {
final RenderDataClient stackDataClient = defaultDataClient.buildClient(trimStackId.getOwner(),
trimStackId.getProject());
stackDataClient.setStackState(trimStackId.getStack(), StackMetaData.StackState.COMPLETE);
}

LOG.info("completeTrimStacks: exit");
}

private static final Logger LOG = LoggerFactory.getLogger(MFOVMontageMatchPatchClient.class);
}