From 5da8e989b5810208488411ccc5f1c178dac77fac Mon Sep 17 00:00:00 2001 From: scheres Date: Mon, 4 Apr 2022 16:17:34 +0100 Subject: [PATCH 001/495] started to make a tomography version of exp_model --- src/tomo_model.cpp | 33 +++++++++++++ src/tomo_model.h | 119 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 152 insertions(+) create mode 100644 src/tomo_model.cpp create mode 100644 src/tomo_model.h diff --git a/src/tomo_model.cpp b/src/tomo_model.cpp new file mode 100644 index 000000000..eb8dcce8b --- /dev/null +++ b/src/tomo_model.cpp @@ -0,0 +1,33 @@ +/*************************************************************************** + * + * Author: "Sjors H.W. Scheres" + * MRC Laboratory of Molecular Biology + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * This complete copyright notice must be included in any revised version of the + * source code. Additional authorship citations may be added, but existing + * author citations must be preserved. + ***************************************************************************/ + +#include "src/tomo_model.h" +bool TomographyExperiment::read(FileName fn_in, int verb) +{ + + if (!fn_in.isStarFile()) + { + return false; + } + + MetaDataTable MDoptics, MDglobal; + MDoptics.read(fn_in, "optics"); + +} diff --git a/src/tomo_model.h b/src/tomo_model.h new file mode 100644 index 000000000..95a91b8dd --- /dev/null +++ b/src/tomo_model.h @@ -0,0 +1,119 @@ +/*************************************************************************** + * + * Author: "Sjors H.W. Scheres" + * MRC Laboratory of Molecular Biology + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * This complete copyright notice must be included in any revised version of the + * source code. Additional authorship citations may be added, but existing + * author citations must be preserved. + ***************************************************************************/ + +#ifndef RELION_TOMO_MODEL_H +#define RELION_TOMO_MODEL_H + +#include "src/metadata_table.h" +#include + +class TomographyExperiment +{ +public: + // All groups in the experiment + std::vector tomograms; + + // Observation model holding the data for all optics groups + ObservationModel obsModel; + + // Empty Constructor + TomographyExperiment() + { + clear(); + } + + ~TomographyExperiment() + { + clear(); + } + + void clear() + { + tomograms.clear(); + } + + // Calculate the total number of tomograms in this tomography experiment + long int numberOfTomograms() + { + return tomograms.size(); + } + + // Calculate the total number of tilt series images in this tomography experiment + long int numberOfTiltSeriesImages(); + + // Calculate the total number of optics groups in this experiment + int numberOfOpticsGroups() + { + return obsModel.numberOfOpticsGroups(); + } + + // Get the pixel size for this optics group + RFLOAT getOpticsPixelSize(int optics_group); + + // Get the original image size for this optics group + int getOpticsImageSize(int optics_group); + + // Get the random_subset for this particle + int getRandomSubset(long int part_id); + + // Get the group_id for the N'th image for this particle + long int getGroupId(long int part_id, int img_id); + + // Get the optics group to which the N'th image for this particle belongs + int getOpticsGroup(long int part_id, int img_id); + + // Get the original position in the input STAR file for the N'th image for this particle + int getOriginalImageId(long int part_id, int img_id); + + // Get the pixel size for the N-th image of this particle + RFLOAT getImagePixelSize(long int part_id, int img_id); + + // Get the vector of number of images per group_id + void getNumberOfImagesPerGroup(std::vector &nr_particles_per_group, int random_subset = 0); + + // Get the vector of number of images per group_id + void getNumberOfImagesPerOpticsGroup(std::vector &nr_particles_per_group, int random_subset = 0); + + // Get the metadata-row for this image in a separate MetaDataTable + MetaDataTable getMetaDataImage(long int part_id, int img_id); + + // Which micrograph (or tomogram) doe this particle image comes from? + FileName getMicrographName(long int ori_img_id); + FileName getMicrographName(long int part_id, int img_id); + + // Add a particle + long int addParticle(std::string part_name, int random_subset = 0); + + // Add an image to the given particle + int addImageToParticle(long int part_id, std::string img_name, long int ori_img_id, long int group_id, + int optics_group, bool unique); + + // Add a group + long int addGroup(std::string mic_name, int optics_group); + + // Read from file + void read(FileName fn_in, int verb = 0); + + // Write to file + void write(FileName fn_root); + +}; + +#endif //RELION_TOMO_MODEL_H From 03c0dc5625ccde3ededdb420307ab28d4e722b96 Mon Sep 17 00:00:00 2001 From: scheres Date: Tue, 5 Apr 2022 17:11:38 +0100 Subject: [PATCH 002/495] started tomo_model --- src/star_handling.cpp | 331 ++++++++++++++++++++++++++++++++++++++++++ src/star_handling.h | 36 +++++ src/tomo_model.cpp | 135 ++++++++++++++++- src/tomo_model.h | 170 +++++++++++++++------- 4 files changed, 615 insertions(+), 57 deletions(-) create mode 100644 src/star_handling.cpp create mode 100644 src/star_handling.h diff --git a/src/star_handling.cpp b/src/star_handling.cpp new file mode 100644 index 000000000..db258b1d3 --- /dev/null +++ b/src/star_handling.cpp @@ -0,0 +1,331 @@ +/*************************************************************************** + * + * Author: "Sjors H.W. Scheres" + * MRC Laboratory of Molecular Biology + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * This complete copyright notice must be included in any revised version of the + * source code. Additional authorship citations may be added, but existing + * author citations must be preserved. + ***************************************************************************/ +#include "src/star_handling.h" + +void combineStarfiles(std::vector &fns_in, + MetaDataTable &MDout, ObservationModel &obsModel, + FileName fn_check, bool do_ignore_optics, std::string tablename_in, int verb) +{ + MetaDataTable MDin0; + std::vector MDsin, MDoptics; + std::vector obsModels; + ObservationModel myobsModel0; + + // Read the first table into the global obsModel + if (do_ignore_optics) MDin0.read(fns_in[0], tablename_in); + else ObservationModel::loadSafely(fns_in[0], obsModel, MDin0, "discover", 1); + MDsin.push_back(MDin0); + + // Read all the rest of the tables into local obsModels + for (int i = 1; i < fns_in.size(); i++) + { + ObservationModel myobsModel; + MetaDataTable MDin; // define again, as reading from previous one may linger here... + if (do_ignore_optics) MDin.read(fns_in[i], tablename_in); + else ObservationModel::loadSafely(fns_in[i], myobsModel, MDin, "discover", 1); + MDsin.push_back(MDin); + obsModels.push_back(myobsModel); + } + + // Combine optics groups with the same EMDL_IMAGE_OPTICS_GROUP_NAME, make new ones for those with a different name + if (!do_ignore_optics) + { + std::vector optics_group_uniq_names; + + // Initialise optics_group_uniq_names with the first table + FOR_ALL_OBJECTS_IN_METADATA_TABLE(obsModel.opticsMdt) + { + std::string myname; + obsModel.opticsMdt.getValue(EMDL_IMAGE_OPTICS_GROUP_NAME, myname); + optics_group_uniq_names.push_back(myname); + } + + // Now check uniqueness of the other tables + for (int MDs_id = 1; MDs_id < fns_in.size(); MDs_id++) + { + const int obs_id = MDs_id - 1; + + std::vector new_optics_groups; + FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDsin[MDs_id]) + { + int tmp; + MDsin[MDs_id].getValue(EMDL_IMAGE_OPTICS_GROUP, tmp); + new_optics_groups.push_back(tmp); + } + + MetaDataTable unique_opticsMdt; + unique_opticsMdt.addMissingLabels(&obsModels[obs_id].opticsMdt); + + FOR_ALL_OBJECTS_IN_METADATA_TABLE(obsModels[obs_id].opticsMdt) + { + std::string myname; + int my_optics_group; + obsModels[obs_id].opticsMdt.getValue(EMDL_IMAGE_OPTICS_GROUP_NAME, myname); + obsModels[obs_id].opticsMdt.getValue(EMDL_IMAGE_OPTICS_GROUP, my_optics_group); + + // Check whether this name is unique + bool is_uniq = true; + int new_group; + for (new_group = 0; new_group < optics_group_uniq_names.size(); new_group++) + { + if (optics_group_uniq_names[new_group] == myname) + { + is_uniq = false; + break; + } + } + new_group ++; // start counting of groups at 1, not 0! + + if (is_uniq) + { + if (verb > 0) std::cout << " + Adding new optics_group with name: " << myname << std::endl; + + optics_group_uniq_names.push_back(myname); + // Add the line to the global obsModel + obsModels[obs_id].opticsMdt.setValue(EMDL_IMAGE_OPTICS_GROUP, new_group); + + unique_opticsMdt.addObject(); + unique_opticsMdt.setObject(obsModels[obs_id].opticsMdt.getObject()); + } + else + { + if (verb > 0) std::cout << " + Joining optics_groups with the same name: " << myname << std::endl; + if (verb > 0) std::cerr << " + WARNING: if these are different data sets, you might want to rename optics groups instead of joining them!" << std::endl; + if (verb > 0) std::cerr << " + WARNING: if so, manually edit the rlnOpticsGroupName column in the optics_groups table of your input STAR files." << std::endl; + } + + if (my_optics_group != new_group) + { + if (verb > 0) std::cout << " + Renumbering group " << myname << " from " << my_optics_group << " to " << new_group << std::endl; + } + + // Update the optics_group entry for all particles in the MDsin + for (long int current_object2 = MDsin[MDs_id].firstObject(); + current_object2 < MDsin[MDs_id].numberOfObjects() && current_object2 >= 0; + current_object2 = MDsin[MDs_id].nextObject()) + { + int old_optics_group; + MDsin[MDs_id].getValue(EMDL_IMAGE_OPTICS_GROUP, old_optics_group, current_object2); + if (old_optics_group == my_optics_group) + new_optics_groups[current_object2] = new_group; + } + } + + obsModels[obs_id].opticsMdt = unique_opticsMdt; + + FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDsin[MDs_id]) + { + MDsin[MDs_id].setValue(EMDL_IMAGE_OPTICS_GROUP, new_optics_groups[current_object]); + + // Also rename the rlnGroupName to not have groups overlapping from different optics groups + std::string name; + if (MDsin[MDs_id].getValue(EMDL_MLMODEL_GROUP_NAME, name)) + { + name = "optics"+integerToString(new_optics_groups[current_object])+"_"+name; + MDsin[MDs_id].setValue(EMDL_MLMODEL_GROUP_NAME, name); + } + } + } + + // Make one vector for combination of the optics tables + MDoptics.push_back(obsModel.opticsMdt); + for (int i = 1; i < fns_in.size(); i++) + { + MDoptics.push_back(obsModels[i - 1].opticsMdt); + } + + // Check if anisotropic magnification and/or beam_tilt are present in some optics groups, but not in others. + // If so, initialise the others correctly + bool has_beamtilt = false, has_not_beamtilt = false; + bool has_anisomag = false, has_not_anisomag = false; + bool has_odd_zernike = false, has_not_odd_zernike = false; + bool has_even_zernike = false, has_not_even_zernike = false; + bool has_ctf_premultiplied = false, has_not_ctf_premultiplied = false; + for (int i = 0; i < fns_in.size(); i++) + { + if (MDoptics[i].containsLabel(EMDL_IMAGE_BEAMTILT_X) || + MDoptics[i].containsLabel(EMDL_IMAGE_BEAMTILT_Y)) + { + has_beamtilt = true; + } + else + { + has_not_beamtilt = true; + } + if (MDoptics[i].containsLabel(EMDL_IMAGE_MAG_MATRIX_00) && + MDoptics[i].containsLabel(EMDL_IMAGE_MAG_MATRIX_01) && + MDoptics[i].containsLabel(EMDL_IMAGE_MAG_MATRIX_10) && + MDoptics[i].containsLabel(EMDL_IMAGE_MAG_MATRIX_11)) + { + has_anisomag = true; + } + else + { + has_not_anisomag = true; + } + if (MDoptics[i].containsLabel(EMDL_IMAGE_ODD_ZERNIKE_COEFFS)) + { + has_odd_zernike = true; + } + else + { + has_not_odd_zernike = true; + } + if (MDoptics[i].containsLabel(EMDL_IMAGE_EVEN_ZERNIKE_COEFFS)) + { + has_even_zernike = true; + } + else + { + has_not_even_zernike = true; + } + if (MDoptics[i].containsLabel(EMDL_OPTIMISER_DATA_ARE_CTF_PREMULTIPLIED)) + { + has_ctf_premultiplied = true; + } + else + { + has_not_ctf_premultiplied = true; + } + } +#ifdef DEBUG + printf("has_beamtilt = %d, has_not_beamtilt = %d, has_anisomag = %d, has_not_anisomag = %d, has_odd_zernike = %d, has_not_odd_zernike = %d, has_even_zernike = %d, has_not_even_zernike = %d, has_ctf_premultiplied = %d, has_not_ctf_premultiplied = %d\n", has_beamtilt, has_not_beamtilt, has_anisomag, has_not_anisomag, has_odd_zernike, has_not_odd_zernike, has_even_zernike, has_not_even_zernike, has_ctf_premultiplied, has_not_ctf_premultiplied); +#endif + + for (int i = 0; i < fns_in.size(); i++) + { + if (has_beamtilt && has_not_beamtilt) + { + if (!MDoptics[i].containsLabel(EMDL_IMAGE_BEAMTILT_X)) + { + FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDoptics[i]) + { + MDoptics[i].setValue(EMDL_IMAGE_BEAMTILT_X, 0.); + } + } + if (!MDoptics[i].containsLabel(EMDL_IMAGE_BEAMTILT_Y)) + { + FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDoptics[i]) + { + MDoptics[i].setValue(EMDL_IMAGE_BEAMTILT_Y, 0.); + } + } + } + + if (has_anisomag && has_not_anisomag) + { + if (!(MDoptics[i].containsLabel(EMDL_IMAGE_MAG_MATRIX_00) && + MDoptics[i].containsLabel(EMDL_IMAGE_MAG_MATRIX_01) && + MDoptics[i].containsLabel(EMDL_IMAGE_MAG_MATRIX_10) && + MDoptics[i].containsLabel(EMDL_IMAGE_MAG_MATRIX_11)) ) + { + FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDoptics[i]) + { + MDoptics[i].setValue(EMDL_IMAGE_MAG_MATRIX_00, 1.); + MDoptics[i].setValue(EMDL_IMAGE_MAG_MATRIX_01, 0.); + MDoptics[i].setValue(EMDL_IMAGE_MAG_MATRIX_10, 0.); + MDoptics[i].setValue(EMDL_IMAGE_MAG_MATRIX_11, 1.); + } + } + } + + if (has_odd_zernike && has_not_odd_zernike) + { + std::vector six_zeros(6, 0); + if (!MDoptics[i].containsLabel(EMDL_IMAGE_ODD_ZERNIKE_COEFFS)) + { + FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDoptics[i]) + { + MDoptics[i].setValue(EMDL_IMAGE_ODD_ZERNIKE_COEFFS, six_zeros); + } + } + } + + if (has_even_zernike && has_not_even_zernike) + { + std::vector nine_zeros(9, 0); + if (!MDoptics[i].containsLabel(EMDL_IMAGE_EVEN_ZERNIKE_COEFFS)) + { + FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDoptics[i]) + { + MDoptics[i].setValue(EMDL_IMAGE_EVEN_ZERNIKE_COEFFS, nine_zeros); + } + } + } + + if (has_ctf_premultiplied && has_not_ctf_premultiplied) + { + if (!MDoptics[i].containsLabel(EMDL_OPTIMISER_DATA_ARE_CTF_PREMULTIPLIED)) + { + FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDoptics[i]) + { + MDoptics[i].setValue(EMDL_OPTIMISER_DATA_ARE_CTF_PREMULTIPLIED, false); + } + } + } + } + + // Now combine all optics tables into one + obsModel.opticsMdt = MetaDataTable::combineMetaDataTables(MDoptics); + } + + // Combine the particles tables + MDout = MetaDataTable::combineMetaDataTables(MDsin); + + //Deactivate the group_name column + MDout.deactivateLabel(EMDL_MLMODEL_GROUP_NO); + + if (fn_check != "") + { + EMDLabel label = EMDL::str2Label(fn_check); + if (!MDout.containsLabel(label)) + REPORT_ERROR("ERROR: the output file does not contain the label to check for duplicates. Is it present in all input files?"); + + /// Don't want to mess up original order, so make a MDsort with only that label... + FileName fn_this, fn_prev = ""; + MetaDataTable MDsort; + FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDout) + { + MDout.getValue(label, fn_this); + MDsort.addObject(); + MDsort.setValue(label, fn_this); + } + // sort on the label + MDsort.newSort(label); + long int nr_duplicates = 0; + FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDsort) + { + MDsort.getValue(label, fn_this); + if (fn_this == fn_prev) + { + nr_duplicates++; + std::cerr << " WARNING: duplicate entry: " << fn_this << std::endl; + } + fn_prev = fn_this; + } + + if (nr_duplicates > 0) + std::cerr << " WARNING: Total number of duplicate "<< fn_check << " entries: " << nr_duplicates << std::endl; + } + + MDout.setName(MDin0.getName()); + +} + diff --git a/src/star_handling.h b/src/star_handling.h new file mode 100644 index 000000000..bf94ae1f2 --- /dev/null +++ b/src/star_handling.h @@ -0,0 +1,36 @@ +/*************************************************************************** + * + * Author: "Sjors H.W. Scheres" + * MRC Laboratory of Molecular Biology + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * This complete copyright notice must be included in any revised version of the + * source code. Additional authorship citations may be added, but existing + * author citations must be preserved. + ***************************************************************************/ + + +#ifndef RELION_STAR_HANDLING_H +#define RELION_STAR_HANDLING_H + +#include +#include +#include + +// Read in all the STAR files in the vector; if do_ignore_optics is true, use tablename_in, otherwise fill obsModel +// fn_check is a string for duplication checking, leave empty for no checking +void combineStarfiles(std::vector &fns_in, + MetaDataTable &MDout, ObservationModel &obsModel, + FileName fn_check = "", bool do_ignore_optics = false, std::string tablename_in = "", int verb = 0); + + +#endif //RELION_STAR_HANDLING_H diff --git a/src/tomo_model.cpp b/src/tomo_model.cpp index eb8dcce8b..5f8a16517 100644 --- a/src/tomo_model.cpp +++ b/src/tomo_model.cpp @@ -19,15 +19,146 @@ ***************************************************************************/ #include "src/tomo_model.h" + + +RFLOAT ExpTiltSerie::getOpticsPixelSize(int optics_group) +{ + return obsModel.getPixelSize(optics_group); +} + +int ExpTiltSerie::getOpticsImageSize(int optics_group) +{ + return obsModel.getBoxSize(optics_group); +} + + + bool TomographyExperiment::read(FileName fn_in, int verb) { + clear(); + if (!fn_in.isStarFile()) { + if (verb > 0) std::cerr << " ERROR: input filename for TomographyExperiment is not a STAR file" << std::endl; return false; } - MetaDataTable MDoptics, MDglobal; - MDoptics.read(fn_in, "optics"); + MDtiltseries.read(fn_in, "tilt_series"); + if (MDtiltseries.numberOfObjects() == 0) + { + if (verb > 0) std::cerr << " ERROR: input starfile for TomographyExperiment " << fn_in << " does not contain any entries in tilt_series table" << std::endl; + return false; + } + + for (long int ts_id = 0; ts_id < MDtiltseries.numberOfObjects(); ts_id++) + { + ExpTiltSerie mytiltserie; + mytiltserie.id = ts_id; + + FileName fn_star; + MDtiltseries.getValue(EMDL_TOMO_TILT_SERIES_STARFILE, fn_star, ts_id); + MDtiltseries.getValue(EMDL_TOMO_NAME, mytiltserie.name, ts_id); + + ObservationModel::loadSafely(fn_star, mytiltserie.obsModel, MDtiltseries, "movies", verb); + if (mytiltserie.obsModel.opticsMdt.numberOfObjects() == 0) + { + if (verb > 0) std::cerr << " ERROR: input starfile for tilt serie " << fn_star << " does not contain any optics groups" << std::endl; + return false; + } + + // Get all the necessary information about this tiltseries + mytiltserie.MDtiltimages.read(fn_star, "tilt_images"); + for (long int timg_id = 0; timg_id < mytiltserie.MDtiltimages.numberOfObjects(); timg_id++) + { + ExpTiltImage mytiltimage; + mytiltimage.id = timg_id; + mytiltserie.MDtiltimages.getValue(EMDL_IMAGE_OPTICS_GROUP, mytiltimage.optics_group, timg_id); + mytiltimage.tiltseries_id = ts_id; + mytiltserie.tiltimages.push_back(mytiltimage); + } + + tiltseries.push_back(mytiltserie); + + } + + return true; + +} + + +void TomographyExperiment::write(FileName fn_root) +{ + + // Write the tilt_series.star file in the root directory + if(fn_root[fn_root.size()-1]!='/') fn_root+='/'; + FileName fn_out = fn_root + "tilt_series.star"; + MDtiltseries.setName("tilt_series"); + MDtiltseries.write(fn_out); + + // Also write all the star files with the individual starfiles for the tilt images in the root directory + for (long int ts_id = 0; ts_id < MDtiltseries.numberOfObjects(); ts_id++) + { + + FileName fn_star; + MDtiltseries.getValue(EMDL_TOMO_TILT_SERIES_STARFILE, fn_star, ts_id); + FileName fn_newstar = getOutputFileWithNewUniqueDate(fn_star, fn_root); + + // Create output directory if neccesary + FileName newdir = fn_newstar.beforeLastOf("/"); + if (!exists(newdir)) + { + mktree(newdir); + } + + std::ofstream fh; + fh.open((fn_newstar).c_str(), std::ios::out); + if (!fh) + REPORT_ERROR( (std::string)"TomographyExperiment::write: Cannot write file: " + fn_newstar); + + tiltseries[ts_id].obsModel.opticsMdt.setName("optics"); + tiltseries[ts_id].obsModel.opticsMdt.write(fh); + + tiltseries[ts_id].MDtiltimages.setName("tilt_images"); + tiltseries[ts_id].MDtiltimages.write(fh); + + fh.close(); + + } + +} + +long int TomographyExperiment::numberOfTiltImages() +{ + + long int result = 0; + for (long int i = 0; i < tiltseries.size(); i++) + { + result += tiltseries[i].tiltimages.size(); + } + return result; + +} + +void TomographyExperiment::generateSingleMetaDataTable(MetaDataTable &MDout, ObservationModel &obsModel) +{ + std::vector fn_stars; + for (long int ts_id = 0; ts_id < MDtiltseries.numberOfObjects(); ts_id++) + { + FileName fn_star; + MDtiltseries.getValue(EMDL_TOMO_TILT_SERIES_STARFILE, fn_star, ts_id); + fn_stars.push_back(fn_star); + } + + combineStarfiles(fn_stars, MDout, obsModel); + +} + +void TomographyExperiment::convertBackFromSingleMetaDataTable(MetaDataTable &MDin, ObservationModel &obsModel) +{ + for (long int ts_id = 0; ts_id < MDtiltseries.numberOfObjects(); ts_id++) + { + tiltseries[ts_id].MDtiltimages = subsetMetaDataTable(MDin, EMDL_TOMO_NAME, tiltseries[ts_id].name); + } } diff --git a/src/tomo_model.h b/src/tomo_model.h index 95a91b8dd..ec2927dde 100644 --- a/src/tomo_model.h +++ b/src/tomo_model.h @@ -22,43 +22,97 @@ #define RELION_TOMO_MODEL_H #include "src/metadata_table.h" +#include "src/star_handling.h" #include -class TomographyExperiment +class ExpTiltImage { public: - // All groups in the experiment - std::vector tomograms; - // Observation model holding the data for all optics groups - ObservationModel obsModel; + // ID of this tiltimage inside its own tiltserie + long int id; + + // ID of the tiltserie that this tiltimage came from + long int tiltseries_id; + + // Optics group of the image + int optics_group; // Empty Constructor - TomographyExperiment() - { - clear(); - } + ExpTiltImage() {} + + // Destructor needed for work with vectors + ~ExpTiltImage() {} + + // Copy constructor needed for work with vectors + ExpTiltImage(ExpTiltImage const& copy) + { + id = copy.id; + tiltseries_id = copy.tiltseries_id; + optics_group = copy.optics_group; + } + + // Define assignment operator in terms of the copy constructor + ExpTiltImage& operator=(ExpTiltImage const& copy) + { + id = copy.id; + tiltseries_id = copy.tiltseries_id; + optics_group = copy.optics_group; + return *this; + } - ~TomographyExperiment() - { - clear(); - } +}; - void clear() - { - tomograms.clear(); - } +class ExpTiltSerie +{ +public: + // All tiltimages in the tiltserie + std::vector tiltimages; - // Calculate the total number of tomograms in this tomography experiment - long int numberOfTomograms() + // ID of the tiltserie + long int id; + + // Name of the tiltserie (by this name it will be recognised upon reading) + std::string name; + + // Observation model holding the data for all optics groups + ObservationModel obsModel; + + // table with all the metadata + MetaDataTable MDtiltimages; + + // Empty Constructor + ExpTiltSerie() {} + + // Destructor needed for work with vectors + ~ExpTiltSerie() {} + + // Copy constructor needed for work with vectors + ExpTiltSerie(ExpTiltSerie const& copy) + { + tiltimages = copy.tiltimages; + id = copy.id; + name = copy.name; + obsModel = copy.obsModel; + MDtiltimages = copy.MDtiltimages; + } + + // Define assignment operator in terms of the copy constructor + ExpTiltSerie& operator=(ExpTiltSerie const& copy) + { + tiltimages = copy.tiltimages; + id = copy.id; + name = copy.name; + obsModel = copy.obsModel; + MDtiltimages = copy.MDtiltimages; + return *this; + } + long int numberOfTiltImages() { - return tomograms.size(); + return MDtiltimages.numberOfObjects(); } - // Calculate the total number of tilt series images in this tomography experiment - long int numberOfTiltSeriesImages(); - - // Calculate the total number of optics groups in this experiment + // Calculate the total number of optics groups in this tiltserie int numberOfOpticsGroups() { return obsModel.numberOfOpticsGroups(); @@ -70,50 +124,56 @@ class TomographyExperiment // Get the original image size for this optics group int getOpticsImageSize(int optics_group); - // Get the random_subset for this particle - int getRandomSubset(long int part_id); - - // Get the group_id for the N'th image for this particle - long int getGroupId(long int part_id, int img_id); - // Get the optics group to which the N'th image for this particle belongs - int getOpticsGroup(long int part_id, int img_id); +}; - // Get the original position in the input STAR file for the N'th image for this particle - int getOriginalImageId(long int part_id, int img_id); - // Get the pixel size for the N-th image of this particle - RFLOAT getImagePixelSize(long int part_id, int img_id); +class TomographyExperiment +{ +public: + // All tiltseries in the experiment + std::vector tiltseries; - // Get the vector of number of images per group_id - void getNumberOfImagesPerGroup(std::vector &nr_particles_per_group, int random_subset = 0); + // Metadata table with information about all tiltseries + MetaDataTable MDtiltseries; - // Get the vector of number of images per group_id - void getNumberOfImagesPerOpticsGroup(std::vector &nr_particles_per_group, int random_subset = 0); + // Empty Constructor + TomographyExperiment() + { + clear(); + } - // Get the metadata-row for this image in a separate MetaDataTable - MetaDataTable getMetaDataImage(long int part_id, int img_id); + ~TomographyExperiment() + { + clear(); + } - // Which micrograph (or tomogram) doe this particle image comes from? - FileName getMicrographName(long int ori_img_id); - FileName getMicrographName(long int part_id, int img_id); + void clear() + { + tiltseries.clear(); + MDtiltseries.clear(); + } - // Add a particle - long int addParticle(std::string part_name, int random_subset = 0); + // Read from file, return false if this is not a TomographyExperiment + bool read(FileName fn_in, int verb = 0); - // Add an image to the given particle - int addImageToParticle(long int part_id, std::string img_name, long int ori_img_id, long int group_id, - int optics_group, bool unique); + // Write to file + void write(FileName fn_root); - // Add a group - long int addGroup(std::string mic_name, int optics_group); + // Calculate the total number of tiltseries in this tomography experiment + long int numberOfTiltseries() + { + return tiltseries.size(); + } - // Read from file - void read(FileName fn_in, int verb = 0); + // Calculate the total number of tilt series images in this tomography experiment + long int numberOfTiltImages(); - // Write to file - void write(FileName fn_root); + // Make one big metadatatable with all movies/micrographs (to be used for motioncorr and ctffind runners) + void generateSingleMetaDataTable(MetaDataTable &MDout, ObservationModel &obsModel); + // Convert back from one big metadatatable into separate STAR files for each tilt serie + void convertBackFromSingleMetaDataTable(MetaDataTable &MDin, ObservationModel &obsModel); }; #endif //RELION_TOMO_MODEL_H From c697d3dcba732fc5909f169aed9f619b8916998a Mon Sep 17 00:00:00 2001 From: scheres Date: Tue, 5 Apr 2022 18:46:18 +0100 Subject: [PATCH 003/495] keep on working on initial tomo_model --- src/apps/star_handler.cpp | 311 +------------------------- src/jaz/single_particle/obs_model.cpp | 46 ++++ src/jaz/single_particle/obs_model.h | 4 + src/metadata_label.h | 2 + src/metadata_table.cpp | 3 + src/metadata_table.h | 1 + src/star_handling.cpp | 1 - src/tomo_model.cpp | 42 ++-- 8 files changed, 82 insertions(+), 328 deletions(-) diff --git a/src/apps/star_handler.cpp b/src/apps/star_handler.cpp index 3d47c8877..e1b9b3276 100644 --- a/src/apps/star_handler.cpp +++ b/src/apps/star_handler.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -260,7 +261,10 @@ class star_handler_parameters else MDout = subsetMetaDataTable(MDin, EMDL::str2Label(select_str_label), select_exclude_str, true); - write_check_ignore_optics(MDout, fn_out, MDin.getName()); + // Remove optics groups that are no longer in use, and renumber in MDout + obsModel.removeUnusedOpticsGroups(MDout); + + write_check_ignore_optics(MDout, fn_out, MDin.getName()); std::cout << " Written: " << fn_out << std::endl; } @@ -341,7 +345,7 @@ class star_handler_parameters std::vector fns_in; std::vector words; tokenize(fn_in, words); - for (int iword = 0; iword < words.size(); iword++) + for (int iword = 0; iword < words.size(); iword++) { FileName fnt = words[iword]; const int n_files = fns_in.size(); @@ -353,307 +357,10 @@ class star_handler_parameters if (fns_in.size() == 0) REPORT_ERROR("No STAR files to combine."); - MetaDataTable MDin0, MDout; - std::vector MDsin, MDoptics; - std::vector obsModels; - ObservationModel myobsModel0; - // Read the first table into the global obsModel - if (do_ignore_optics) MDin0.read(fns_in[0], tablename_in); - else ObservationModel::loadSafely(fns_in[0], obsModel, MDin0, "discover", 1); - MDsin.push_back(MDin0); - // Read all the rest of the tables into local obsModels - for (int i = 1; i < fns_in.size(); i++) - { - ObservationModel myobsModel; - MetaDataTable MDin; // define again, as reading from previous one may linger here... - if (do_ignore_optics) MDin.read(fns_in[i], tablename_in); - else ObservationModel::loadSafely(fns_in[i], myobsModel, MDin, "discover", 1); - MDsin.push_back(MDin); - obsModels.push_back(myobsModel); - } - - // Combine optics groups with the same EMDL_IMAGE_OPTICS_GROUP_NAME, make new ones for those with a different name - if (!do_ignore_optics) - { - std::vector optics_group_uniq_names; - - // Initialise optics_group_uniq_names with the first table - FOR_ALL_OBJECTS_IN_METADATA_TABLE(obsModel.opticsMdt) - { - std::string myname; - obsModel.opticsMdt.getValue(EMDL_IMAGE_OPTICS_GROUP_NAME, myname); - optics_group_uniq_names.push_back(myname); - } - - // Now check uniqueness of the other tables - for (int MDs_id = 1; MDs_id < fns_in.size(); MDs_id++) - { - const int obs_id = MDs_id - 1; - - std::vector new_optics_groups; - FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDsin[MDs_id]) - { - int tmp; - MDsin[MDs_id].getValue(EMDL_IMAGE_OPTICS_GROUP, tmp); - new_optics_groups.push_back(tmp); - } - - MetaDataTable unique_opticsMdt; - unique_opticsMdt.addMissingLabels(&obsModels[obs_id].opticsMdt); - - FOR_ALL_OBJECTS_IN_METADATA_TABLE(obsModels[obs_id].opticsMdt) - { - std::string myname; - int my_optics_group; - obsModels[obs_id].opticsMdt.getValue(EMDL_IMAGE_OPTICS_GROUP_NAME, myname); - obsModels[obs_id].opticsMdt.getValue(EMDL_IMAGE_OPTICS_GROUP, my_optics_group); - - // Check whether this name is unique - bool is_uniq = true; - int new_group; - for (new_group = 0; new_group < optics_group_uniq_names.size(); new_group++) - { - if (optics_group_uniq_names[new_group] == myname) - { - is_uniq = false; - break; - } - } - new_group ++; // start counting of groups at 1, not 0! - - if (is_uniq) - { - std::cout << " + Adding new optics_group with name: " << myname << std::endl; - - optics_group_uniq_names.push_back(myname); - // Add the line to the global obsModel - obsModels[obs_id].opticsMdt.setValue(EMDL_IMAGE_OPTICS_GROUP, new_group); - - unique_opticsMdt.addObject(); - unique_opticsMdt.setObject(obsModels[obs_id].opticsMdt.getObject()); - } - else - { - std::cout << " + Joining optics_groups with the same name: " << myname << std::endl; - std::cerr << " + WARNING: if these are different data sets, you might want to rename optics groups instead of joining them!" << std::endl; - std::cerr << " + WARNING: if so, manually edit the rlnOpticsGroupName column in the optics_groups table of your input STAR files." << std::endl; - } - - if (my_optics_group != new_group) - { - std::cout << " + Renumbering group " << myname << " from " << my_optics_group << " to " << new_group << std::endl; - } - - // Update the optics_group entry for all particles in the MDsin - for (long int current_object2 = MDsin[MDs_id].firstObject(); - current_object2 < MDsin[MDs_id].numberOfObjects() && current_object2 >= 0; - current_object2 = MDsin[MDs_id].nextObject()) - { - int old_optics_group; - MDsin[MDs_id].getValue(EMDL_IMAGE_OPTICS_GROUP, old_optics_group, current_object2); - if (old_optics_group == my_optics_group) - new_optics_groups[current_object2] = new_group; - } - } - - obsModels[obs_id].opticsMdt = unique_opticsMdt; - - FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDsin[MDs_id]) - { - MDsin[MDs_id].setValue(EMDL_IMAGE_OPTICS_GROUP, new_optics_groups[current_object]); - - // Also rename the rlnGroupName to not have groups overlapping from different optics groups - std::string name; - if (MDsin[MDs_id].getValue(EMDL_MLMODEL_GROUP_NAME, name)) - { - name = "optics"+integerToString(new_optics_groups[current_object])+"_"+name; - MDsin[MDs_id].setValue(EMDL_MLMODEL_GROUP_NAME, name); - } - } - } - - // Make one vector for combination of the optics tables - MDoptics.push_back(obsModel.opticsMdt); - for (int i = 1; i < fns_in.size(); i++) - { - MDoptics.push_back(obsModels[i - 1].opticsMdt); - } - - // Check if anisotropic magnification and/or beam_tilt are present in some optics groups, but not in others. - // If so, initialise the others correctly - bool has_beamtilt = false, has_not_beamtilt = false; - bool has_anisomag = false, has_not_anisomag = false; - bool has_odd_zernike = false, has_not_odd_zernike = false; - bool has_even_zernike = false, has_not_even_zernike = false; - bool has_ctf_premultiplied = false, has_not_ctf_premultiplied = false; - for (int i = 0; i < fns_in.size(); i++) - { - if (MDoptics[i].containsLabel(EMDL_IMAGE_BEAMTILT_X) || - MDoptics[i].containsLabel(EMDL_IMAGE_BEAMTILT_Y)) - { - has_beamtilt = true; - } - else - { - has_not_beamtilt = true; - } - if (MDoptics[i].containsLabel(EMDL_IMAGE_MAG_MATRIX_00) && - MDoptics[i].containsLabel(EMDL_IMAGE_MAG_MATRIX_01) && - MDoptics[i].containsLabel(EMDL_IMAGE_MAG_MATRIX_10) && - MDoptics[i].containsLabel(EMDL_IMAGE_MAG_MATRIX_11)) - { - has_anisomag = true; - } - else - { - has_not_anisomag = true; - } - if (MDoptics[i].containsLabel(EMDL_IMAGE_ODD_ZERNIKE_COEFFS)) - { - has_odd_zernike = true; - } - else - { - has_not_odd_zernike = true; - } - if (MDoptics[i].containsLabel(EMDL_IMAGE_EVEN_ZERNIKE_COEFFS)) - { - has_even_zernike = true; - } - else - { - has_not_even_zernike = true; - } - if (MDoptics[i].containsLabel(EMDL_OPTIMISER_DATA_ARE_CTF_PREMULTIPLIED)) - { - has_ctf_premultiplied = true; - } - else - { - has_not_ctf_premultiplied = true; - } - } -#ifdef DEBUG - printf("has_beamtilt = %d, has_not_beamtilt = %d, has_anisomag = %d, has_not_anisomag = %d, has_odd_zernike = %d, has_not_odd_zernike = %d, has_even_zernike = %d, has_not_even_zernike = %d, has_ctf_premultiplied = %d, has_not_ctf_premultiplied = %d\n", has_beamtilt, has_not_beamtilt, has_anisomag, has_not_anisomag, has_odd_zernike, has_not_odd_zernike, has_even_zernike, has_not_even_zernike, has_ctf_premultiplied, has_not_ctf_premultiplied); -#endif - - for (int i = 0; i < fns_in.size(); i++) - { - if (has_beamtilt && has_not_beamtilt) - { - if (!MDoptics[i].containsLabel(EMDL_IMAGE_BEAMTILT_X)) - { - FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDoptics[i]) - { - MDoptics[i].setValue(EMDL_IMAGE_BEAMTILT_X, 0.); - } - } - if (!MDoptics[i].containsLabel(EMDL_IMAGE_BEAMTILT_Y)) - { - FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDoptics[i]) - { - MDoptics[i].setValue(EMDL_IMAGE_BEAMTILT_Y, 0.); - } - } - } - - if (has_anisomag && has_not_anisomag) - { - if (!(MDoptics[i].containsLabel(EMDL_IMAGE_MAG_MATRIX_00) && - MDoptics[i].containsLabel(EMDL_IMAGE_MAG_MATRIX_01) && - MDoptics[i].containsLabel(EMDL_IMAGE_MAG_MATRIX_10) && - MDoptics[i].containsLabel(EMDL_IMAGE_MAG_MATRIX_11)) ) - { - FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDoptics[i]) - { - MDoptics[i].setValue(EMDL_IMAGE_MAG_MATRIX_00, 1.); - MDoptics[i].setValue(EMDL_IMAGE_MAG_MATRIX_01, 0.); - MDoptics[i].setValue(EMDL_IMAGE_MAG_MATRIX_10, 0.); - MDoptics[i].setValue(EMDL_IMAGE_MAG_MATRIX_11, 1.); - } - } - } - - if (has_odd_zernike && has_not_odd_zernike) - { - std::vector six_zeros(6, 0); - if (!MDoptics[i].containsLabel(EMDL_IMAGE_ODD_ZERNIKE_COEFFS)) - { - FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDoptics[i]) - { - MDoptics[i].setValue(EMDL_IMAGE_ODD_ZERNIKE_COEFFS, six_zeros); - } - } - } - - if (has_even_zernike && has_not_even_zernike) - { - std::vector nine_zeros(9, 0); - if (!MDoptics[i].containsLabel(EMDL_IMAGE_EVEN_ZERNIKE_COEFFS)) - { - FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDoptics[i]) - { - MDoptics[i].setValue(EMDL_IMAGE_EVEN_ZERNIKE_COEFFS, nine_zeros); - } - } - } - - if (has_ctf_premultiplied && has_not_ctf_premultiplied) - { - if (!MDoptics[i].containsLabel(EMDL_OPTIMISER_DATA_ARE_CTF_PREMULTIPLIED)) - { - FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDoptics[i]) - { - MDoptics[i].setValue(EMDL_OPTIMISER_DATA_ARE_CTF_PREMULTIPLIED, false); - } - } - } - } - - // Now combine all optics tables into one - obsModel.opticsMdt = MetaDataTable::combineMetaDataTables(MDoptics); - } - - // Combine the particles tables - MDout = MetaDataTable::combineMetaDataTables(MDsin); - - //Deactivate the group_name column - MDout.deactivateLabel(EMDL_MLMODEL_GROUP_NO); - - if (fn_check != "") - { - EMDLabel label = EMDL::str2Label(fn_check); - if (!MDout.containsLabel(label)) - REPORT_ERROR("ERROR: the output file does not contain the label to check for duplicates. Is it present in all input files?"); - - /// Don't want to mess up original order, so make a MDsort with only that label... - FileName fn_this, fn_prev = ""; - MetaDataTable MDsort; - FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDout) - { - MDout.getValue(label, fn_this); - MDsort.addObject(); - MDsort.setValue(label, fn_this); - } - // sort on the label - MDsort.newSort(label); - long int nr_duplicates = 0; - FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDsort) - { - MDsort.getValue(label, fn_this); - if (fn_this == fn_prev) - { - nr_duplicates++; - std::cerr << " WARNING: duplicate entry: " << fn_this << std::endl; - } - fn_prev = fn_this; - } - - if (nr_duplicates > 0) - std::cerr << " WARNING: Total number of duplicate "<< fn_check << " entries: " << nr_duplicates << std::endl; - } + MetaDataTable MDout; + combineStarfiles(fns_in, MDout, obsModel, fn_check, do_ignore_optics, tablename_in, 1); - write_check_ignore_optics(MDout, fn_out, MDin0.getName()); + write_check_ignore_optics(MDout, fn_out, MDout.getName()); std::cout << " Written: " << fn_out << std::endl; } diff --git a/src/jaz/single_particle/obs_model.cpp b/src/jaz/single_particle/obs_model.cpp index c2f490886..c09f35201 100644 --- a/src/jaz/single_particle/obs_model.cpp +++ b/src/jaz/single_particle/obs_model.cpp @@ -931,6 +931,52 @@ void ObservationModel::sortOpticsGroups(MetaDataTable& partMdt) } } +void ObservationModel::removeUnusedOpticsGroups(MetaDataTable& partMdt) +{ + std::set usedGroups; + for (long int i = 0; i < partMdt.numberOfObjects(); i++) + { + int og; + partMdt.getValue(EMDL_IMAGE_OPTICS_GROUP, og, i); + if (usedGroups.find(og) == usedGroups.end()) + { + usedGroups.insert(og); + } + } + + // Now renumber the remaining optics groups + std::map old2new; + int found = 0; + for (int i = 0; i < opticsMdt.numberOfObjects(); i++) + { + int og; + opticsMdt.getValue(EMDL_IMAGE_OPTICS_GROUP, og, i); + if (usedGroups.find(og) == usedGroups.end()) + { + old2new[og] = -1; + } + else + { + old2new[og] = found + 1; + found++; + } + + opticsMdt.setValue(EMDL_IMAGE_OPTICS_GROUP, old2new[og], i); + } + + // Remove all unused optics groups from the opticsMdt + opticsMdt = subsetMetaDataTable(opticsMdt, EMDL_IMAGE_OPTICS_GROUP, 1, 99999); + + // Change all optics_groups entries in the particle table + for (long int i = 0; i < partMdt.numberOfObjects(); i++) + { + int og; + partMdt.getValue(EMDL_IMAGE_OPTICS_GROUP, og, i); + partMdt.setValue(EMDL_IMAGE_OPTICS_GROUP, old2new[og], i); + } + +} + std::vector ObservationModel::getOptGroupsPresent_oneBased(const MetaDataTable& partMdt) const { const int gc = opticsMdt.numberOfObjects(); diff --git a/src/jaz/single_particle/obs_model.h b/src/jaz/single_particle/obs_model.h index 90810e734..5a49251ef 100644 --- a/src/jaz/single_particle/obs_model.h +++ b/src/jaz/single_particle/obs_model.h @@ -219,6 +219,10 @@ class ObservationModel (Merely changing the order in opticsMdt would fail if groups were missing.) */ void sortOpticsGroups(MetaDataTable& partMdt); + /* Remove unused optics groups, e.g. after a subset of the particles have been selected + and also renumber the groups in the particle table partMdt */ + void removeUnusedOpticsGroups(MetaDataTable& partMdt); + /* Return the set of optics groups present in partMdt */ std::vector getOptGroupsPresent_oneBased(const MetaDataTable& partMdt) const; diff --git a/src/metadata_label.h b/src/metadata_label.h index 5648b7834..e235687a5 100644 --- a/src/metadata_label.h +++ b/src/metadata_label.h @@ -604,6 +604,7 @@ enum EMDLabel EMDL_TOMO_NAME, EMDL_TOMO_TILT_SERIES_NAME, + EMDL_TOMO_TILT_SERIES_STARFILE, EMDL_TOMO_FRAME_COUNT, EMDL_TOMO_SIZE_X, EMDL_TOMO_SIZE_Y, @@ -1286,6 +1287,7 @@ class StaticInitialization EMDL::addLabel(EMDL_TOMO_NAME, EMDL_STRING, "rlnTomoName", "Arbitrary name for a tomogram"); EMDL::addLabel(EMDL_TOMO_TILT_SERIES_NAME, EMDL_STRING, "rlnTomoTiltSeriesName", "Tilt series file name"); + EMDL::addLabel(EMDL_TOMO_TILT_SERIES_STARFILE, EMDL_STRING, "rlnTomoTiltSeriesStarFile", "Tilt series starfile"); EMDL::addLabel(EMDL_TOMO_FRAME_COUNT, EMDL_INT, "rlnTomoFrameCount", "Number of tilts in a tilt series"); EMDL::addLabel(EMDL_TOMO_SIZE_X, EMDL_INT, "rlnTomoSizeX", "Width of a bin-1 tomogram in pixels"); EMDL::addLabel(EMDL_TOMO_SIZE_Y, EMDL_INT, "rlnTomoSizeY", "Height of a bin-1 tomogram in pixels"); diff --git a/src/metadata_table.cpp b/src/metadata_table.cpp index 3a9814468..a85094565 100644 --- a/src/metadata_table.cpp +++ b/src/metadata_table.cpp @@ -1978,6 +1978,7 @@ MetaDataTable subsetMetaDataTable(MetaDataTable &MDin, EMDLabel label, std::stri REPORT_ERROR("subsetMetadataTable ERROR: input MetaDataTable does not contain label: " + EMDL::label2Str(label)); MetaDataTable MDout; + MDout.setName(MDin.getName()); FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDin) { std::string val; @@ -2096,3 +2097,5 @@ MetaDataTable removeDuplicatedParticles(MetaDataTable &MDin, EMDLabel mic_label, return MDout; } + + diff --git a/src/metadata_table.h b/src/metadata_table.h index 7f780f43b..88602eb07 100644 --- a/src/metadata_table.h +++ b/src/metadata_table.h @@ -501,4 +501,5 @@ bool MetaDataTable::setValue(EMDLabel label, const T &value, long int objectID) } } + #endif diff --git a/src/star_handling.cpp b/src/star_handling.cpp index db258b1d3..e2f95c12b 100644 --- a/src/star_handling.cpp +++ b/src/star_handling.cpp @@ -328,4 +328,3 @@ void combineStarfiles(std::vector &fns_in, MDout.setName(MDin0.getName()); } - diff --git a/src/tomo_model.cpp b/src/tomo_model.cpp index 5f8a16517..8e6096fab 100644 --- a/src/tomo_model.cpp +++ b/src/tomo_model.cpp @@ -45,6 +45,7 @@ bool TomographyExperiment::read(FileName fn_in, int verb) } MDtiltseries.read(fn_in, "tilt_series"); + if (MDtiltseries.numberOfObjects() == 0) { if (verb > 0) std::cerr << " ERROR: input starfile for TomographyExperiment " << fn_in << " does not contain any entries in tilt_series table" << std::endl; @@ -60,7 +61,7 @@ bool TomographyExperiment::read(FileName fn_in, int verb) MDtiltseries.getValue(EMDL_TOMO_TILT_SERIES_STARFILE, fn_star, ts_id); MDtiltseries.getValue(EMDL_TOMO_NAME, mytiltserie.name, ts_id); - ObservationModel::loadSafely(fn_star, mytiltserie.obsModel, MDtiltseries, "movies", verb); + ObservationModel::loadSafely(fn_star, mytiltserie.obsModel, mytiltserie.MDtiltimages, "movies", verb); if (mytiltserie.obsModel.opticsMdt.numberOfObjects() == 0) { if (verb > 0) std::cerr << " ERROR: input starfile for tilt serie " << fn_star << " does not contain any optics groups" << std::endl; @@ -68,7 +69,6 @@ bool TomographyExperiment::read(FileName fn_in, int verb) } // Get all the necessary information about this tiltseries - mytiltserie.MDtiltimages.read(fn_star, "tilt_images"); for (long int timg_id = 0; timg_id < mytiltserie.MDtiltimages.numberOfObjects(); timg_id++) { ExpTiltImage mytiltimage; @@ -87,23 +87,19 @@ bool TomographyExperiment::read(FileName fn_in, int verb) } -void TomographyExperiment::write(FileName fn_root) +void TomographyExperiment::write(FileName fn_outdir) { + // Make user fn_outdir ends in a slah: it should be a directory + if (fn_outdir[fn_outdir.size()-1]!='/') fn_outdir+='/'; - // Write the tilt_series.star file in the root directory - - if(fn_root[fn_root.size()-1]!='/') fn_root+='/'; - FileName fn_out = fn_root + "tilt_series.star"; - MDtiltseries.setName("tilt_series"); - MDtiltseries.write(fn_out); - - // Also write all the star files with the individual starfiles for the tilt images in the root directory + // Write all the star files with the individual starfiles for the tilt images in the ouput directory for (long int ts_id = 0; ts_id < MDtiltseries.numberOfObjects(); ts_id++) { FileName fn_star; MDtiltseries.getValue(EMDL_TOMO_TILT_SERIES_STARFILE, fn_star, ts_id); - FileName fn_newstar = getOutputFileWithNewUniqueDate(fn_star, fn_root); + FileName fn_newstar = getOutputFileWithNewUniqueDate(fn_star, fn_outdir); + MDtiltseries.setValue(EMDL_TOMO_TILT_SERIES_STARFILE, fn_newstar, ts_id); // Create output directory if neccesary FileName newdir = fn_newstar.beforeLastOf("/"); @@ -112,21 +108,15 @@ void TomographyExperiment::write(FileName fn_root) mktree(newdir); } - std::ofstream fh; - fh.open((fn_newstar).c_str(), std::ios::out); - if (!fh) - REPORT_ERROR( (std::string)"TomographyExperiment::write: Cannot write file: " + fn_newstar); - - tiltseries[ts_id].obsModel.opticsMdt.setName("optics"); - tiltseries[ts_id].obsModel.opticsMdt.write(fh); - - tiltseries[ts_id].MDtiltimages.setName("tilt_images"); - tiltseries[ts_id].MDtiltimages.write(fh); - - fh.close(); + tiltseries[ts_id].obsModel.save(tiltseries[ts_id].MDtiltimages, fn_newstar, "tilt_images"); } + // Also write the (now modified with fn_newstars) tilt_series.star file in the root directory + FileName fn_out = fn_outdir + "tilt_series.star"; + MDtiltseries.setName("tilt_series"); + MDtiltseries.write(fn_out); + } long int TomographyExperiment::numberOfTiltImages() @@ -159,6 +149,8 @@ void TomographyExperiment::convertBackFromSingleMetaDataTable(MetaDataTable &MDi { for (long int ts_id = 0; ts_id < MDtiltseries.numberOfObjects(); ts_id++) { - tiltseries[ts_id].MDtiltimages = subsetMetaDataTable(MDin, EMDL_TOMO_NAME, tiltseries[ts_id].name); + tiltseries[ts_id].obsModel = obsModel; + tiltseries[ts_id].MDtiltimages = subsetMetaDataTable(MDin, EMDL_TOMO_NAME, tiltseries[ts_id].name, false); + tiltseries[ts_id].obsModel.removeUnusedOpticsGroups(tiltseries[ts_id].MDtiltimages); } } From 7f562f916f0428ffbd191fc7fca4edebb258c6b6 Mon Sep 17 00:00:00 2001 From: scheres Date: Wed, 6 Apr 2022 14:55:19 +0100 Subject: [PATCH 004/495] inserted tomography model into motioncorr_runner, also do pre-exposure dose-weighting in motioncorr_runner --- src/motioncorr_runner.cpp | 75 ++++++++++++++++++++++++++++++----- src/motioncorr_runner.h | 17 +++++++- src/motioncorr_runner_mpi.cpp | 7 ++-- src/tomo_model.cpp | 58 +++++++++++++++++++++------ src/tomo_model.h | 8 ++-- 5 files changed, 133 insertions(+), 32 deletions(-) diff --git a/src/motioncorr_runner.cpp b/src/motioncorr_runner.cpp index ed8305589..49fda1548 100644 --- a/src/motioncorr_runner.cpp +++ b/src/motioncorr_runner.cpp @@ -78,7 +78,7 @@ void MotioncorrRunner::read(int argc, char **argv, int rank) { parser.setCommandLine(argc, argv); int gen_section = parser.addSection("General options"); - fn_in = parser.getOption("--i", "STAR file with all input micrographs, or a Linux wildcard with all micrographs to operate on"); + fn_in = parser.getOption("--i", "STAR file with all input micrographs, the tomography tilt series, or a Linux wildcard with all micrographs to operate on"); fn_out = parser.getOption("--o", "Name for the output directory", "MotionCorr"); do_skip_logfile = parser.checkOption("--skip_logfile", "Skip generation of tracks-part of the logfile.pdf"); n_threads = textToInteger(parser.getOption("--j", "Number of threads per movie (= process)", "1")); @@ -233,14 +233,29 @@ void MotioncorrRunner::initialise() #endif // Set up which micrograph movies to process + is_tomo = false; if (fn_in.isStarFile()) { - MetaDataTable MDin; - ObservationModel::loadSafely(fn_in, obsModel, MDin, "movies", verb); - if (MDin.numberOfObjects() > 0 && !MDin.containsLabel(EMDL_MICROGRAPH_MOVIE_NAME)) + MetaDataTable MDin; + + // Check if this is a TomographyExperiment starfile, and if so, unpack into one large metadatatable + if (tomo_model.read(fn_in, 1)) + { + is_tomo = true; + tomo_model.generateSingleMetaDataTable(MDin, obsModel); + } + else + { + ObservationModel::loadSafely(fn_in, obsModel, MDin, "movies", verb); + } + + + if (MDin.numberOfObjects() > 0 && !MDin.containsLabel(EMDL_MICROGRAPH_MOVIE_NAME)) REPORT_ERROR("The input STAR file does not contain the rlnMicrographMovieName column. Are you sure you imported files as movies, not single frame images?"); fn_micrographs.clear(); + fn_tomogram_names.clear(); + pre_exposure_micrographs.clear(); optics_group_micrographs.clear(); FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDin) { @@ -251,6 +266,26 @@ void MotioncorrRunner::initialise() int optics_group; MDin.getValue(EMDL_IMAGE_OPTICS_GROUP, optics_group); optics_group_micrographs.push_back(optics_group); + + RFLOAT my_pre_exposure; + if (MDin.getValue(EMDL_MICROGRAPH_PRE_EXPOSURE, my_pre_exposure)) + { + pre_exposure_micrographs.push_back(my_pre_exposure); + } + else + { + pre_exposure_micrographs.push_back(0.0); + } + + if (is_tomo) + { + FileName fn_tomo; + MDin.getValue(EMDL_TOMO_NAME, fn_tomo); + fn_tomogram_names.push_back(fn_tomo); + long index; + MDin.getValue(EMDL_TOMO_TILT_MOVIE_INDEX, index); + tomo_tilt_movie_index.push_back(index); + } } } else @@ -465,7 +500,10 @@ void MotioncorrRunner::run() Micrograph mic(fn_micrographs[imic], fn_gain_reference, bin_factor, eer_upsampling, eer_grouping); - // Get angpix and voltage from the optics groups: + // Set per-micrograph pre_exposure + mic.pre_exposure = pre_exposure + pre_exposure_micrographs[imic]; + + // Get angpix and voltage from the optics groups: obsModel.opticsMdt.getValue(EMDL_CTF_VOLTAGE, voltage, optics_group_micrographs[imic]-1); obsModel.opticsMdt.getValue(EMDL_MICROGRAPH_ORIGINAL_PIXEL_SIZE, angpix, optics_group_micrographs[imic]-1); @@ -555,7 +593,7 @@ bool MotioncorrRunner::executeMotioncor2(Micrograph &mic, int rank) { command += " -Kv " + floatToString(voltage); command += " -FmDose " + floatToString(dose_per_frame); - command += " -InitDose " + floatToString(pre_exposure); + command += " -InitDose " + floatToString(mic.pre_exposure); } if (fn_defect != "") @@ -812,7 +850,6 @@ void MotioncorrRunner::saveModel(Micrograph &mic) { mic.angpix = angpix; mic.voltage = voltage; mic.dose_per_frame = dose_per_frame; - mic.pre_exposure = pre_exposure; mic.fnDefect = fn_defect; FileName fn_avg = getOutputFileNames(mic.getMovieFilename()); @@ -850,7 +887,12 @@ void MotioncorrRunner::generateLogFilePDFAndWriteStarFiles() { MDavg.setValue(EMDL_CTF_POWER_SPECTRUM, fn_avg.withoutExtension() + "_PS.mrc"); } - MDavg.setValue(EMDL_MICROGRAPH_NAME, fn_avg); + if (is_tomo) + { + MDavg.setValue(EMDL_TOMO_NAME, fn_tomogram_names[imic]); + MDavg.setValue(EMDL_TOMO_TILT_MOVIE_INDEX, tomo_tilt_movie_index[imic]); + } + MDavg.setValue(EMDL_MICROGRAPH_NAME, fn_avg); MDavg.setValue(EMDL_MICROGRAPH_METADATA_NAME, fn_avg.withoutExtension() + ".star"); MDavg.setValue(EMDL_IMAGE_OPTICS_GROUP, optics_group_ori_micrographs[imic]); FileName fn_star = fn_avg.withoutExtension() + ".star"; @@ -904,13 +946,22 @@ void MotioncorrRunner::generateLogFilePDFAndWriteStarFiles() my_angpix *= bin_factor; obsModel.opticsMdt.setValue(EMDL_MICROGRAPH_PIXEL_SIZE, my_angpix); } - obsModel.save(MDavg, fn_out + "corrected_micrographs.star", "micrographs"); + if (is_tomo) + { + tomo_model.convertBackFromSingleMetaDataTable(MDavg, obsModel); + tomo_model.write(fn_out); + } + else + { + obsModel.save(MDavg, fn_out + "corrected_micrographs.star", "micrographs"); + } if (verb > 0 ) { progress_bar(fn_ori_micrographs.size()); - std::cout << " Done! Written: " << fn_out << "corrected_micrographs.star" << std::endl; + if (is_tomo) std::cout << " Done! Written: " << fn_out << "tilt_series.star" << std::endl; + else std::cout << " Done! Written: " << fn_out << "corrected_micrographs.star" << std::endl; } if (verb > 0) std::cout << " Now generating logfile.pdf ... " << std::endl; @@ -1640,10 +1691,12 @@ bool MotioncorrRunner::executeOwnMotionCorrection(Micrograph &mic) { REPORT_ERROR("Sorry, dose weighting is supported only for 300, 200 or 100 kV"); } + logfile << "Pre-exposure: = " << pre_exposure << std::endl; + std::vector doses(n_frames); for (int iframe = 0; iframe < n_frames; iframe++) { // dose AFTER each frame. - doses[iframe] = pre_exposure + dose_per_frame * (frames[iframe] + 1); + doses[iframe] = mic.pre_exposure + dose_per_frame * (frames[iframe] + 1); if (std::abs(voltage - 200) <= 2) { doses[iframe] /= 0.8; // 200 kV electron is more damaging. } else if (std::abs(voltage - 100) <= 2) { diff --git a/src/motioncorr_runner.h b/src/motioncorr_runner.h index 30ee5c368..2ea13ccec 100644 --- a/src/motioncorr_runner.h +++ b/src/motioncorr_runner.h @@ -31,6 +31,7 @@ #include "src/metadata_table.h" #include "src/image.h" #include "src/micrograph_model.h" +#include "src/tomo_model.h" #include class MotioncorrRunner @@ -51,15 +52,27 @@ class MotioncorrRunner FileName fn_in, fn_out, fn_movie; // Filenames of all the micrographs to run Motioncorr on - std::vector fn_micrographs, fn_ori_micrographs; + std::vector fn_micrographs, fn_ori_micrographs, fn_tomogram_names; // Optics group number for all original micrographs std::vector optics_group_micrographs, optics_group_ori_micrographs; + // Pre-exposure for each micrograph (mainly used for tomography) + std::vector pre_exposure_micrographs; + + // Tilt movie index for each micrograph in a tilt serie (needed for converting back to tomographyExperiment) + std::vector tomo_tilt_movie_index; + // Information about the optics groups ObservationModel obsModel; - // Skip generation of logfile + // Is this a tomography experiment? + bool is_tomo; + + // Information about tomography experiment + TomographyExperiment tomo_model; + + // Skip generation of logfile bool do_skip_logfile; // Use our own implementation diff --git a/src/motioncorr_runner_mpi.cpp b/src/motioncorr_runner_mpi.cpp index e2a2c006a..773765047 100644 --- a/src/motioncorr_runner_mpi.cpp +++ b/src/motioncorr_runner_mpi.cpp @@ -44,7 +44,7 @@ void MotioncorrRunnerMpi::run() divide_equally(fn_micrographs.size(), node->size, node->rank, my_first_micrograph, my_last_micrograph); my_nr_micrographs = my_last_micrograph - my_first_micrograph + 1; - int barstep; + int barstep; if (verb > 0) { if (do_own) @@ -52,7 +52,7 @@ void MotioncorrRunnerMpi::run() else if (do_motioncor2) std::cout << " Correcting beam-induced motions using Shawn Zheng's MOTIONCOR2 ..." << std::endl; else - REPORT_ERROR("Bug: by now it should be clear whether to use MotionCor2 or Unblur..."); + REPORT_ERROR("Bug: by now it should be clear whether to use MotionCor2 or our own implementation..."); init_progress_bar(my_nr_micrographs); barstep = XMIPP_MAX(1, my_nr_micrographs / 60); @@ -68,8 +68,9 @@ void MotioncorrRunnerMpi::run() MPI_Abort(MPI_COMM_WORLD, RELION_EXIT_ABORTED); Micrograph mic(fn_micrographs[imic], fn_gain_reference, bin_factor, eer_upsampling, eer_grouping); + mic.pre_exposure = pre_exposure + pre_exposure_micrographs[imic]; - // Get angpix and voltage from the optics groups: + // Get angpix and voltage from the optics groups: obsModel.opticsMdt.getValue(EMDL_CTF_VOLTAGE, voltage, optics_group_micrographs[imic]-1); obsModel.opticsMdt.getValue(EMDL_MICROGRAPH_ORIGINAL_PIXEL_SIZE, angpix, optics_group_micrographs[imic]-1); diff --git a/src/tomo_model.cpp b/src/tomo_model.cpp index 8e6096fab..01dd4e383 100644 --- a/src/tomo_model.cpp +++ b/src/tomo_model.cpp @@ -45,11 +45,19 @@ bool TomographyExperiment::read(FileName fn_in, int verb) } MDtiltseries.read(fn_in, "tilt_series"); - if (MDtiltseries.numberOfObjects() == 0) { - if (verb > 0) std::cerr << " ERROR: input starfile for TomographyExperiment " << fn_in << " does not contain any entries in tilt_series table" << std::endl; - return false; + return false; + } + + if (!MDtiltseries.containsLabel(EMDL_TOMO_TILT_SERIES_STARFILE)) + { + REPORT_ERROR("ERROR: input starfile for TomographyExperiment " + fn_in + " does not contain rlnTomoTiltSeriesStarFile label "); + } + + if (!MDtiltseries.containsLabel(EMDL_TOMO_NAME)) + { + REPORT_ERROR("ERROR: input starfile for TomographyExperiment " + fn_in + " does not contain rlnTomoName label "); } for (long int ts_id = 0; ts_id < MDtiltseries.numberOfObjects(); ts_id++) @@ -64,10 +72,17 @@ bool TomographyExperiment::read(FileName fn_in, int verb) ObservationModel::loadSafely(fn_star, mytiltserie.obsModel, mytiltserie.MDtiltimages, "movies", verb); if (mytiltserie.obsModel.opticsMdt.numberOfObjects() == 0) { - if (verb > 0) std::cerr << " ERROR: input starfile for tilt serie " << fn_star << " does not contain any optics groups" << std::endl; - return false; + REPORT_ERROR("ERROR: input starfile for tilt series " + fn_star + " does not contain any optics groups"); } + if (!mytiltserie.MDtiltimages.containsLabel(EMDL_TOMO_TILT_MOVIE_INDEX)) + { + REPORT_ERROR("ERROR: input starfile for tilt series " + fn_star + " does not contain rlnTomoTiltMovieIndex label "); + } + + // Make sure tilt images are sorted on their index (used to convert back from single large metadatatable) + mytiltserie.MDtiltimages.newSort(EMDL_TOMO_TILT_MOVIE_INDEX); + /* // Get all the necessary information about this tiltseries for (long int timg_id = 0; timg_id < mytiltserie.MDtiltimages.numberOfObjects(); timg_id++) { @@ -77,7 +92,7 @@ bool TomographyExperiment::read(FileName fn_in, int verb) mytiltimage.tiltseries_id = ts_id; mytiltserie.tiltimages.push_back(mytiltimage); } - + */ tiltseries.push_back(mytiltserie); } @@ -125,7 +140,7 @@ long int TomographyExperiment::numberOfTiltImages() long int result = 0; for (long int i = 0; i < tiltseries.size(); i++) { - result += tiltseries[i].tiltimages.size(); + result += tiltseries[i].numberOfTiltImages(); } return result; @@ -147,10 +162,27 @@ void TomographyExperiment::generateSingleMetaDataTable(MetaDataTable &MDout, Obs void TomographyExperiment::convertBackFromSingleMetaDataTable(MetaDataTable &MDin, ObservationModel &obsModel) { - for (long int ts_id = 0; ts_id < MDtiltseries.numberOfObjects(); ts_id++) - { - tiltseries[ts_id].obsModel = obsModel; - tiltseries[ts_id].MDtiltimages = subsetMetaDataTable(MDin, EMDL_TOMO_NAME, tiltseries[ts_id].name, false); - tiltseries[ts_id].obsModel.removeUnusedOpticsGroups(tiltseries[ts_id].MDtiltimages); - } + if (!MDin.containsLabel(EMDL_TOMO_TILT_MOVIE_INDEX)) + { + REPORT_ERROR("BUG: the MDin that is passed to TomographyExperiment::convertBackFromSingleMetaDataTable should contain a rlnTomoTiltMovieIndex label"); + } + + for (long int ts_id = 0; ts_id < MDtiltseries.numberOfObjects(); ts_id++) + { + tiltseries[ts_id].obsModel = obsModel; + MetaDataTable MDjoin = subsetMetaDataTable(MDin, EMDL_TOMO_NAME, tiltseries[ts_id].name, false); + MDjoin.newSort(EMDL_TOMO_TILT_MOVIE_INDEX); + if (MDjoin.numberOfObjects() != tiltseries[ts_id].MDtiltimages.numberOfObjects()) + { + std::cerr << "MDjoin= "; MDjoin.write(std::cerr); + std::cerr << "tiltseries[ts_id].MDtiltimages= "; tiltseries[ts_id].MDtiltimages.write(std::cerr); + REPORT_ERROR("ERROR: unequal number of Objects in tiltserie starfiles"); + } + FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDjoin) + { + //MetaDataContainer* mine = MDjoin.getObject(); + tiltseries[ts_id].MDtiltimages.setObject(MDjoin.getObject(), current_object); + } + tiltseries[ts_id].obsModel.removeUnusedOpticsGroups(tiltseries[ts_id].MDtiltimages); + } } diff --git a/src/tomo_model.h b/src/tomo_model.h index ec2927dde..d3fa72925 100644 --- a/src/tomo_model.h +++ b/src/tomo_model.h @@ -25,6 +25,7 @@ #include "src/star_handling.h" #include +/* class ExpTiltImage { public: @@ -62,12 +63,13 @@ class ExpTiltImage } }; +*/ class ExpTiltSerie { public: // All tiltimages in the tiltserie - std::vector tiltimages; + //std::vector tiltimages; // ID of the tiltserie long int id; @@ -90,7 +92,7 @@ class ExpTiltSerie // Copy constructor needed for work with vectors ExpTiltSerie(ExpTiltSerie const& copy) { - tiltimages = copy.tiltimages; + //tiltimages = copy.tiltimages; id = copy.id; name = copy.name; obsModel = copy.obsModel; @@ -100,7 +102,7 @@ class ExpTiltSerie // Define assignment operator in terms of the copy constructor ExpTiltSerie& operator=(ExpTiltSerie const& copy) { - tiltimages = copy.tiltimages; + //tiltimages = copy.tiltimages; id = copy.id; name = copy.name; obsModel = copy.obsModel; From 0bb240411300a2c580ab542e93b032571f374a86 Mon Sep 17 00:00:00 2001 From: scheres Date: Wed, 6 Apr 2022 16:49:22 +0100 Subject: [PATCH 005/495] also added tomo_model support to run_ctffind --- src/ctffind_runner.cpp | 47 +++++++++++++++++++++++++-- src/ctffind_runner.h | 13 ++++++++ src/jaz/single_particle/obs_model.cpp | 6 ++++ src/motioncorr_runner.cpp | 6 +++- src/tomo_model.cpp | 3 +- 5 files changed, 70 insertions(+), 5 deletions(-) diff --git a/src/ctffind_runner.cpp b/src/ctffind_runner.cpp index cfc17b703..b64dfe714 100644 --- a/src/ctffind_runner.cpp +++ b/src/ctffind_runner.cpp @@ -140,10 +140,25 @@ void CtffindRunner::initialise(bool is_leader) fn_out += "/"; // Set up which micrographs to estimate CTFs from - if (fn_in.isStarFile()) + is_tomo = false; + if (fn_in.isStarFile()) { MetaDataTable MDin; - ObservationModel::loadSafely(fn_in, obsModel, MDin, "micrographs", verb); + + // Check if this is a TomographyExperiment starfile, and if so, unpack into one large metadatatable + if (tomo_model.read(fn_in, 1)) + { + is_tomo = true; + tomo_model.generateSingleMetaDataTable(MDin, obsModel); + } + else + { + ObservationModel::loadSafely(fn_in, obsModel, MDin, "micrographs", verb); + } + if (MDin.numberOfObjects() == 0) + { + REPORT_ERROR("ERROR: no input micrographs to work on."); + } if (MDin.numberOfObjects() > 0 && !MDin.containsLabel(EMDL_MICROGRAPH_NAME)) REPORT_ERROR("ERROR: There is no rlnMicrographName label in the input micrograph STAR file."); @@ -173,6 +188,17 @@ void CtffindRunner::initialise(bool is_leader) int optics_group; MDin.getValue(EMDL_IMAGE_OPTICS_GROUP, optics_group); optics_group_micrographs_all.push_back(optics_group); + + if (is_tomo) + { + FileName fn_tomo; + MDin.getValue(EMDL_TOMO_NAME, fn_tomo); + fn_tomogram_names.push_back(fn_tomo); + long index; + MDin.getValue(EMDL_TOMO_TILT_MOVIE_INDEX, index); + tomo_tilt_movie_index.push_back(index); + } + } } else @@ -469,12 +495,27 @@ void CtffindRunner::joinCtffindResults() MDctf.setValue(EMDL_CTF_PHASESHIFT, phaseshift); if (fabs(valscore + 999.) > 0.) MDctf.setValue(EMDL_CTF_VALIDATIONSCORE, valscore); + + if (is_tomo) + { + MDctf.setValue(EMDL_TOMO_NAME, fn_tomogram_names[imic]); + MDctf.setValue(EMDL_TOMO_TILT_MOVIE_INDEX, tomo_tilt_movie_index[imic]); + } + } if (verb > 0 && imic % 60 == 0) progress_bar(imic); } - obsModel.save(MDctf, fn_out+"micrographs_ctf.star", "micrographs"); + if (is_tomo) + { + tomo_model.convertBackFromSingleMetaDataTable(MDctf, obsModel); + tomo_model.write(fn_out); + } + else + { + obsModel.save(MDctf, fn_out + "micrographs_ctf.star", "micrographs"); + } if (verb > 0) { diff --git a/src/ctffind_runner.h b/src/ctffind_runner.h index 0ec1b6441..4309f32fe 100644 --- a/src/ctffind_runner.h +++ b/src/ctffind_runner.h @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -57,6 +58,18 @@ class CtffindRunner // Information about the optics groups ObservationModel obsModel; + // Tilt movie index for each micrograph in a tilt serie (needed for converting back to tomographyExperiment) + std::vector tomo_tilt_movie_index; + + // Tomogram names of all the micrographs + std::vector fn_tomogram_names; + + // Is this a tomography experiment? + bool is_tomo; + + // Information about tomography experiment + TomographyExperiment tomo_model; + // Dimension of squared area of the micrograph to use for CTF estimation int ctf_win; diff --git a/src/jaz/single_particle/obs_model.cpp b/src/jaz/single_particle/obs_model.cpp index c09f35201..c1ab5d9b5 100644 --- a/src/jaz/single_particle/obs_model.cpp +++ b/src/jaz/single_particle/obs_model.cpp @@ -1337,6 +1337,10 @@ void ObservationModel::loadSafely(std::string filename, ObservationModel& obsMod { mytablename = "movies"; } + else if (particlesMdt.read(filename, "tilt_images")) + { + mytablename = "tilt_images"; + } } else { @@ -1365,6 +1369,8 @@ void ObservationModel::loadSafely(std::string filename, ObservationModel& obsMod particlesMdt.setName("particles"); else if (particlesMdt.containsLabel(EMDL_MICROGRAPH_MOVIE_NAME)) particlesMdt.setName("movies"); + else if (particlesMdt.containsLabel(EMDL_TOMO_TILT_MOVIE_INDEX)) + particlesMdt.setName("tilt_images"); else particlesMdt.setName("micrographs"); } diff --git a/src/motioncorr_runner.cpp b/src/motioncorr_runner.cpp index 49fda1548..70ec15269 100644 --- a/src/motioncorr_runner.cpp +++ b/src/motioncorr_runner.cpp @@ -241,6 +241,7 @@ void MotioncorrRunner::initialise() // Check if this is a TomographyExperiment starfile, and if so, unpack into one large metadatatable if (tomo_model.read(fn_in, 1)) { + std::cerr <<"is_tomo"< 0 && !MDin.containsLabel(EMDL_MICROGRAPH_MOVIE_NAME)) REPORT_ERROR("The input STAR file does not contain the rlnMicrographMovieName column. Are you sure you imported files as movies, not single frame images?"); diff --git a/src/tomo_model.cpp b/src/tomo_model.cpp index 01dd4e383..34983156f 100644 --- a/src/tomo_model.cpp +++ b/src/tomo_model.cpp @@ -69,13 +69,14 @@ bool TomographyExperiment::read(FileName fn_in, int verb) MDtiltseries.getValue(EMDL_TOMO_TILT_SERIES_STARFILE, fn_star, ts_id); MDtiltseries.getValue(EMDL_TOMO_NAME, mytiltserie.name, ts_id); - ObservationModel::loadSafely(fn_star, mytiltserie.obsModel, mytiltserie.MDtiltimages, "movies", verb); + ObservationModel::loadSafely(fn_star, mytiltserie.obsModel, mytiltserie.MDtiltimages, "tilt_images", verb); if (mytiltserie.obsModel.opticsMdt.numberOfObjects() == 0) { REPORT_ERROR("ERROR: input starfile for tilt series " + fn_star + " does not contain any optics groups"); } if (!mytiltserie.MDtiltimages.containsLabel(EMDL_TOMO_TILT_MOVIE_INDEX)) { + std::cerr << "table= "; mytiltserie.MDtiltimages.write(std::cerr); REPORT_ERROR("ERROR: input starfile for tilt series " + fn_star + " does not contain rlnTomoTiltMovieIndex label "); } From 0d4c0338648180bd153b35be19096f8b73f18d25 Mon Sep 17 00:00:00 2001 From: scheres Date: Wed, 6 Apr 2022 16:50:17 +0100 Subject: [PATCH 006/495] add motioncorr and ctffind jobtypes to the tomo GUI. Note that Import is no longer good for those --- src/gui_jobwindow.cpp | 6 +- src/gui_mainwindow.cpp | 14 +++++ src/pipeline_jobs.cpp | 122 ++++++++++++++++++++++++++++++----------- src/pipeline_jobs.h | 10 +++- 4 files changed, 114 insertions(+), 38 deletions(-) diff --git a/src/gui_jobwindow.cpp b/src/gui_jobwindow.cpp index 61d14f5e1..66e8d4230 100644 --- a/src/gui_jobwindow.cpp +++ b/src/gui_jobwindow.cpp @@ -521,7 +521,8 @@ void JobWindow::initialiseMotioncorrWindow() tab1->label("I/O"); resetHeight(); - place("input_star_mics", TOGGLE_DEACTIVATE); + if (is_tomo) place("input_tilt_series", TOGGLE_DEACTIVATE); + else place("input_star_mics", TOGGLE_DEACTIVATE); // Add a little spacer current_y += STEPY/2; @@ -589,7 +590,8 @@ void JobWindow::initialiseCtffindWindow() tab1->label("I/O"); resetHeight(); - place("input_star_mics", TOGGLE_DEACTIVATE); + if (is_tomo) place("input_tilt_series", TOGGLE_DEACTIVATE); + else place("input_star_mics", TOGGLE_DEACTIVATE); place("use_noDW", TOGGLE_DEACTIVATE); // Add a little spacer diff --git a/src/gui_mainwindow.cpp b/src/gui_mainwindow.cpp index 107ec8598..c79c06df3 100644 --- a/src/gui_mainwindow.cpp +++ b/src/gui_mainwindow.cpp @@ -415,6 +415,20 @@ GuiMainWindow::GuiMainWindow(int w, int h, const char* title, FileName fn_pipe, browse_grp[nr_browse_tabs]->end(); nr_browse_tabs++; + browse_grp[nr_browse_tabs] = new Fl_Group(WCOL0, 2, 550, 615-MENUHEIGHT); + browser->add("Motion correction"); + gui_jobwindows[nr_browse_tabs] = new JobWindow(); + gui_jobwindows[nr_browse_tabs]->initialise(PROC_MOTIONCORR, _do_tomo); + browse_grp[nr_browse_tabs]->end(); + nr_browse_tabs++; + + browse_grp[nr_browse_tabs] = new Fl_Group(WCOL0, 2, 550, 615-MENUHEIGHT); + browser->add("CTF estimation"); + gui_jobwindows[nr_browse_tabs] = new JobWindow(); + gui_jobwindows[nr_browse_tabs]->initialise(PROC_CTFFIND, _do_tomo); + browse_grp[nr_browse_tabs]->end(); + nr_browse_tabs++; + browse_grp[nr_browse_tabs] = new Fl_Group(WCOL0, 2, 550, 615-MENUHEIGHT); browser->add("Make pseudo-subtomos"); gui_jobwindows[nr_browse_tabs] = new JobWindow(); diff --git a/src/pipeline_jobs.cpp b/src/pipeline_jobs.cpp index 7ef16aafb..2ce581551 100644 --- a/src/pipeline_jobs.cpp +++ b/src/pipeline_jobs.cpp @@ -1365,8 +1365,15 @@ void RelionJob::initialiseMotioncorrJob() { hidden_name = ".gui_motioncorr"; - joboptions["input_star_mics"] = JobOption("Input movies STAR file:", NODE_MOVIES_CPIPE, "", "STAR files (*.star)", "A STAR file with all micrographs to run MOTIONCORR on"); - joboptions["first_frame_sum"] = JobOption("First frame for corrected sum:", 1, 1, 32, 1, "First frame to use in corrected average (starts counting at 1). "); + if (is_tomo) + { + joboptions["input_tilt_series"] = JobOption("Input tilt series set: ", OUTNODE_TOMO_TILT_SERIES, "", "Tilt series set STAR file (tilt_series.star)", "Input tomo tilt series set. Generated from CCPEM Import functionality..."); + } + else + { + joboptions["input_star_mics"] = JobOption("Input movies STAR file:", NODE_MOVIES_CPIPE, "", "STAR files (*.star)", "A STAR file with all micrographs to run MOTIONCORR on"); + } + joboptions["first_frame_sum"] = JobOption("First frame for corrected sum:", 1, 1, 32, 1, "First frame to use in corrected average (starts counting at 1). "); joboptions["last_frame_sum"] = JobOption("Last frame for corrected sum:", -1, 0, 32, 1, "Last frame to use in corrected average. Values equal to or smaller than 0 mean 'use all frames'."); joboptions["eer_grouping"] = JobOption("EER fractionation:", 32, 1, 100, 1, "The number of hardware frames to group into one fraction. This option is relevant only for Falcon4 movies in the EER format. Note that all 'frames' in the GUI (e.g. first and last frame for corrected sum, dose per frame) refer to fractions, not raw detector frames. See https://www3.mrc-lmb.cam.ac.uk/relion/index.php/Image_compression#Falcon4_EER for detailed guidance on EER processing."); joboptions["do_float16"] = JobOption("Write output in float16?", true ,"If set to Yes, RelionCor2 will write output images in float16 MRC format. This will save a factor of two in disk space compared to the default of writing in float32. Note that RELION and CCPEM will read float16 images, but other programs may not (yet) do so. For example, Gctf will not work with float16 images. Also note that this option does not work with UCSF MotionCor2. For CTF estimation, use CTFFIND-4.1 with pre-calculated power spectra (activate the 'Save sum of power spectra' option)."); @@ -1423,21 +1430,41 @@ bool RelionJob::getCommandsMotioncorrJob(std::string &outputname, std::vector 1) + command="`which relion_run_ctffind_mpi`"; + else + command="`which relion_run_ctffind`"; + if (error_message != "") return false; - if (joboptions["nr_mpi"].getNumber(error_message) > 1) - command="`which relion_run_ctffind_mpi`"; - else - command="`which relion_run_ctffind`"; - if (error_message != "") return false; + // I/O + if (is_tomo) + { + if (joboptions["input_tilt_series"].getString() == "") + { + error_message = "ERROR: empty field for input tilt series STAR file..."; + return false; + } + command += " --i " + joboptions["input_tilt_series"].getString(); + Node node(joboptions["input_tilt_series"].getString(), joboptions["input_tilt_series"].node_type); + inputNodes.push_back(node); + } + else + { + if (joboptions["input_star_mics"].getString() == "") + { + error_message = "ERROR: empty field for input STAR file..."; + return false; + } + command += " --i " + joboptions["input_star_mics"].getString(); + Node node(joboptions["input_star_mics"].getString(), joboptions["input_star_mics"].node_type); + inputNodes.push_back(node); + } - command += " --i " + joboptions["input_star_mics"].getString(); command += " --o " + outputname; command += " --Box " + joboptions["box"].getString(); command += " --ResMin " + joboptions["resmin"].getString(); diff --git a/src/pipeline_jobs.h b/src/pipeline_jobs.h index 9e66612df..bbf9c97a0 100644 --- a/src/pipeline_jobs.h +++ b/src/pipeline_jobs.h @@ -200,7 +200,7 @@ static bool do_allow_change_minimum_dedicated; #define NODE_TOMO_TOMOGRAMS_LABEL "relion.TomoTomogramSet" #define NODE_TOMO_TRAJECTORIES_LABEL "relion.TomoTrajectorySet" #define NODE_TOMO_MANIFOLDS_LABEL "relion.TomoManifoldSet" - +#define NODE_TOMO_TILTSERIES_LABEL "relion.TomoTiltSeriesSet" // nodes compatible with the CCPEM pipeliner // General types that are used as input nodes in the pipeliner @@ -333,6 +333,7 @@ static bool do_allow_change_minimum_dedicated; #define OUTNODE_TOMO_HALFMAP 207 #define OUTNODE_TOMO_POST 208 #define OUTNODE_TOMO_POST_LOG 209 +#define OUTNODE_TOMO_TILT_SERIES 210 #define LABEL_MOVIES_CPIPE "MicrographMoviesData.star.relion" @@ -421,6 +422,7 @@ static bool do_allow_change_minimum_dedicated; #define LABEL_TOMO_HALFMAP "DensityMap.mrc.relion.tomo.halfmap" #define LABEL_TOMO_POST "ProcessData.star.relion.tomo.postprocess" #define LABEL_TOMO_POST_LOG "LogFile.pdf.relion.tomo.postprocess" +#define LABEL_TOMO_TILT_SERIES "ProcessData.star.relion.tomo.tiltseries_set" static std::map node_type2pipeliner_label = {{NODE_MOVIES_CPIPE, LABEL_MOVIES_CPIPE}, {NODE_MICS_CPIPE, LABEL_MICS_CPIPE}, @@ -506,7 +508,8 @@ static std::map node_type2pipeliner_label = {{NODE_MOVIES_CPIP {OUTNODE_TOMO_MAP, LABEL_TOMO_MAP}, {OUTNODE_TOMO_HALFMAP, LABEL_TOMO_HALFMAP}, {OUTNODE_TOMO_POST, LABEL_TOMO_POST}, - {OUTNODE_TOMO_POST_LOG, LABEL_TOMO_POST_LOG} }; + {OUTNODE_TOMO_POST_LOG, LABEL_TOMO_POST_LOG}, + {OUTNODE_TOMO_TILT_SERIES, LABEL_TOMO_TILT_SERIES}}; static std::map pipeliner_label2type = {{LABEL_MOVIES_CPIPE, NODE_MOVIES_CPIPE}, @@ -593,7 +596,8 @@ static std::map pipeliner_label2type = {{LABEL_MOVIES_CPIPE, N {LABEL_TOMO_MAP, OUTNODE_TOMO_MAP}, {LABEL_TOMO_HALFMAP, OUTNODE_TOMO_HALFMAP}, {LABEL_TOMO_POST, OUTNODE_TOMO_POST}, - {LABEL_TOMO_POST_LOG, OUTNODE_TOMO_POST_LOG} }; + {LABEL_TOMO_POST_LOG, OUTNODE_TOMO_POST_LOG}, + {LABEL_TOMO_TILT_SERIES, OUTNODE_TOMO_TILT_SERIES}}; //// Conversion dict for CCPEM-pipeliner compatibility static std::map node_label2type = {{NODE_MOVIES_LABEL, NODE_MOVIES}, From 61f64a74f5644f0191ade69d2e436acc2e65c910 Mon Sep 17 00:00:00 2001 From: scheres Date: Wed, 6 Apr 2022 16:53:04 +0100 Subject: [PATCH 007/495] removed Import jobtype from tomo GUI as no longer makes sense with motioncorr and ctffind jobtypes for tomo --- src/gui_mainwindow.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/gui_mainwindow.cpp b/src/gui_mainwindow.cpp index c79c06df3..b2fe45638 100644 --- a/src/gui_mainwindow.cpp +++ b/src/gui_mainwindow.cpp @@ -408,12 +408,14 @@ GuiMainWindow::GuiMainWindow(int w, int h, const char* title, FileName fn_pipe, if (_do_tomo) { - browse_grp[nr_browse_tabs] = new Fl_Group(WCOL0, 2, 550, 615-MENUHEIGHT); + /* SHWS 6Apr2022: with new motioncorr and ctffind jobtypes, the old Import makes no sesne anymore, so removed for now + browse_grp[nr_browse_tabs] = new Fl_Group(WCOL0, 2, 550, 615-MENUHEIGHT); browser->add("Tomo import"); gui_jobwindows[nr_browse_tabs] = new JobWindow(); gui_jobwindows[nr_browse_tabs]->initialise(PROC_TOMO_IMPORT); browse_grp[nr_browse_tabs]->end(); nr_browse_tabs++; + */ browse_grp[nr_browse_tabs] = new Fl_Group(WCOL0, 2, 550, 615-MENUHEIGHT); browser->add("Motion correction"); From 3a4649fc3c5efa9a9d37870a1343894e4fe9ba18 Mon Sep 17 00:00:00 2001 From: scheres Date: Wed, 6 Apr 2022 20:12:35 +0100 Subject: [PATCH 008/495] remove tomo_model files with TomographyExperiment and move everything over to Jasenkos tomogram_set --- src/ctffind_runner.cpp | 8 +- src/ctffind_runner.h | 4 +- src/gui_jobwindow.cpp | 4 +- src/jaz/tomography/tomogram_set.cpp | 223 ++++++++++++++++++++-------- src/jaz/tomography/tomogram_set.h | 31 ++-- src/motioncorr_runner.cpp | 10 +- src/motioncorr_runner.h | 4 +- src/pipeline_jobs.cpp | 20 +-- src/pipeline_jobs.h | 9 +- src/tomo_model.cpp | 189 ----------------------- src/tomo_model.h | 181 ---------------------- 11 files changed, 207 insertions(+), 476 deletions(-) delete mode 100644 src/tomo_model.cpp delete mode 100644 src/tomo_model.h diff --git a/src/ctffind_runner.cpp b/src/ctffind_runner.cpp index b64dfe714..f1f835667 100644 --- a/src/ctffind_runner.cpp +++ b/src/ctffind_runner.cpp @@ -146,10 +146,10 @@ void CtffindRunner::initialise(bool is_leader) MetaDataTable MDin; // Check if this is a TomographyExperiment starfile, and if so, unpack into one large metadatatable - if (tomo_model.read(fn_in, 1)) + if (tomogramSet.read(fn_in, 1)) { is_tomo = true; - tomo_model.generateSingleMetaDataTable(MDin, obsModel); + tomogramSet.generateSingleMetaDataTable(MDin, obsModel); } else { @@ -509,8 +509,8 @@ void CtffindRunner::joinCtffindResults() if (is_tomo) { - tomo_model.convertBackFromSingleMetaDataTable(MDctf, obsModel); - tomo_model.write(fn_out); + tomogramSet.convertBackFromSingleMetaDataTable(MDctf, obsModel); + tomogramSet.write(fn_out); } else { diff --git a/src/ctffind_runner.h b/src/ctffind_runner.h index 4309f32fe..ab73ade18 100644 --- a/src/ctffind_runner.h +++ b/src/ctffind_runner.h @@ -28,10 +28,10 @@ #include #include #include -#include #include #include #include +#include "src/jaz/tomography/tomogram_set.h" class CtffindRunner { @@ -68,7 +68,7 @@ class CtffindRunner bool is_tomo; // Information about tomography experiment - TomographyExperiment tomo_model; + TomogramSet tomogramSet; // Dimension of squared area of the micrograph to use for CTF estimation int ctf_win; diff --git a/src/gui_jobwindow.cpp b/src/gui_jobwindow.cpp index 66e8d4230..fbb4e2a32 100644 --- a/src/gui_jobwindow.cpp +++ b/src/gui_jobwindow.cpp @@ -521,7 +521,7 @@ void JobWindow::initialiseMotioncorrWindow() tab1->label("I/O"); resetHeight(); - if (is_tomo) place("input_tilt_series", TOGGLE_DEACTIVATE); + if (is_tomo) place("input_tomograms", TOGGLE_DEACTIVATE); else place("input_star_mics", TOGGLE_DEACTIVATE); // Add a little spacer @@ -590,7 +590,7 @@ void JobWindow::initialiseCtffindWindow() tab1->label("I/O"); resetHeight(); - if (is_tomo) place("input_tilt_series", TOGGLE_DEACTIVATE); + if (is_tomo) place("input_tomograms", TOGGLE_DEACTIVATE); else place("input_star_mics", TOGGLE_DEACTIVATE); place("use_noDW", TOGGLE_DEACTIVATE); diff --git a/src/jaz/tomography/tomogram_set.cpp b/src/jaz/tomography/tomogram_set.cpp index 0147567e4..aa1e5254f 100644 --- a/src/jaz/tomography/tomogram_set.cpp +++ b/src/jaz/tomography/tomogram_set.cpp @@ -20,50 +20,128 @@ TomogramSet::TomogramSet() TomogramSet::TomogramSet(std::string filename, bool verbose) { - std::ifstream ifs(filename); + if (!read(filename, verbose)) + { + // This may be a tomograms.star file in the old, original relion-4.0 format. Try to convert + // TODO: make conversion from old tomograms.star to new tilt_series.star! + + std::ifstream ifs(filename); + if (!ifs) + { + REPORT_ERROR_STR("TomogramSet::TomogramSet: Unable to read " << filename); + } + else + { + globalTable.readStar(ifs, "global"); + + if (!globalTable.containsLabel(EMDL_TOMO_NAME)) + { + REPORT_ERROR("ERROR: input starfile for TomogramSet " + filename + " does not contain rlnTomoName label "); + } + + // remove information from optics groups and move into globalObsModel + ObservationModel globalObsModel(globalTable); + globalTable.deactivateLabel(EMDL_IMAGE_OPTICS_GROUP_NAME); + globalTable.deactivateLabel(EMDL_IMAGE_OPTICS_GROUP); + globalTable.deactivateLabel(EMDL_CTF_CS); + globalTable.deactivateLabel(EMDL_CTF_VOLTAGE); + globalTable.deactivateLabel(EMDL_CTF_Q0); + + const int tc = globalTable.numberOfObjects(); + + tomogramTables.resize(tc); + tomogramObsModels.resize(tc); + tomogramNames.resize(tc); + + std::vector allTables = MetaDataTable::readAll(ifs, tc+1); + + for (int t = 0; t < tc; t++) + { + const std::string expectedNewName = globalTable.getString(EMDL_TOMO_NAME, t); + const std::string name = allTables[t+1].getName(); + + if (name != expectedNewName) + { + REPORT_ERROR_STR("TomogramSet::TomogramSet: file is corrupted " << filename); + } + + tomogramTables[t] = allTables[t+1]; + tomogramTables[t].setName("tilt_images"); + tomogramObsModels[t] = globalObsModel; + tomogramNames[t] = expectedNewName; + } + } + + } + + globalTable.setName("global"); - bool namesAreOld = false; +} - if (!ifs) - { - REPORT_ERROR_STR("TomogramSet::TomogramSet: Unable to read " << filename); - } - else - { - globalTable.readStar(ifs, "global"); - - const int tc = globalTable.numberOfObjects(); - - tomogramTables.resize(tc); - - std::vector allTables = MetaDataTable::readAll(ifs, tc+1); - - for (int t = 0; t < tc; t++) - { - const std::string expectedOldName = "tomo_" + ZIO::itoa(t); - const std::string expectedNewName = globalTable.getString(EMDL_TOMO_NAME, t); - const std::string name = allTables[t+1].getName(); +bool TomogramSet::read(std::string filename, bool verbose) +{ - if (name == expectedOldName) - { - namesAreOld = true; - } - else if (name != expectedNewName) - { - REPORT_ERROR_STR("TomogramSet::TomogramSet: file is corrupted " << filename); - } + globalTable.read(filename, "global"); - tomogramTables[t] = allTables[t+1]; - tomogramTables[t].setName(expectedNewName); - } - } - - globalTable.setName("global"); + const int tc = globalTable.numberOfObjects(); + + if (tc == 0) return false; + + if (!globalTable.containsLabel(EMDL_TOMO_TILT_SERIES_STARFILE)) + { + std::cerr << "Warning: " << filename + << " does not have rlnTomoTiltSeriesStarFile labels. It may be written in an old format. If so, will try to convert ..." + << std::endl; + return false; + } + + if (!globalTable.containsLabel(EMDL_TOMO_NAME)) + { + REPORT_ERROR("ERROR: input starfile for TomogramSet " + filename + " does not contain rlnTomoName label "); + } + + tomogramTables.resize(tc); + tomogramObsModels.resize(tc); + tomogramNames.resize(tc); + + for (int t = 0; t < tc; t++) + { + tomogramNames[t] = globalTable.getString(EMDL_TOMO_NAME, t); + std::string fn_star = globalTable.getString(EMDL_TOMO_TILT_SERIES_STARFILE, t); + ObservationModel::loadSafely(fn_star, tomogramObsModels[t], tomogramTables[t], "tilt_images", verbose); + // Make sure tilt images are sorted on their index (used to convert back from single large metadatatable) + tomogramTables[t].newSort(EMDL_TOMO_TILT_MOVIE_INDEX); + } + + return true; + +} + +void TomogramSet::write(FileName filename) +{ + FileName fn_outdir = filename.beforeLastOf("/") + "/"; + + const int tc = tomogramTables.size(); + + // Change all the filenames in tomograms.star + for (int t = 0; t < tc; t++) + { + FileName fn_star; + globalTable.getValue(EMDL_TOMO_TILT_SERIES_STARFILE, fn_star, t); + FileName fn_newstar = getOutputFileWithNewUniqueDate(fn_star, fn_outdir); + globalTable.setValue(EMDL_TOMO_TILT_SERIES_STARFILE, fn_newstar, t); + + // Create output directory if necessary + FileName newdir = fn_newstar.beforeLastOf("/"); + if (!exists(newdir)) mktree(newdir); + + // Write the individual tomogram starfile + tomogramObsModels[t].save(tomogramTables[t], fn_newstar, "tilt_images"); + } + + // Also write the (now modified with fn_newstars) tilt_series.star file in the root directory + globalTable.write(filename); - if (verbose && namesAreOld) - { - Log::warn("Tomogram set " + filename + " is out of date. You are recommended to run relion_exp_update_tomogram_set on it."); - } } Tomogram TomogramSet::loadTomogram(int index, bool loadImageData) const @@ -314,31 +392,6 @@ int TomogramSet::size() const return tomogramTables.size(); } -void TomogramSet::write(std::string filename) const -{ - const int tc = tomogramTables.size(); - - if (filename.find_last_of('/') != std::string::npos) - { - std::string path = filename.substr(0, filename.find_last_of('/')); - mktree(path); - } - - std::ofstream ofs(filename); - - if (!ofs) - { - REPORT_ERROR("TomogramSet::write: unable to write to "+filename); - } - - globalTable.write(ofs); - - for (int t = 0; t < tc; t++) - { - tomogramTables[t].write(ofs); - } -} - void TomogramSet::setProjections(int tomogramIndex, const std::vector& proj) { MetaDataTable& m = tomogramTables[tomogramIndex]; @@ -507,3 +560,45 @@ std::string TomogramSet::getOpticsGroupName(int index) const return globalTable.getString(EMDL_IMAGE_OPTICS_GROUP_NAME, index); } } + +void TomogramSet::generateSingleMetaDataTable(MetaDataTable &MDout, ObservationModel &obsModel) +{ + std::vector fn_stars; + for (long int t = 0; t < globalTable.numberOfObjects(); t++) + { + FileName fn_star = globalTable.getString(EMDL_TOMO_TILT_SERIES_STARFILE, t); + fn_stars.push_back(fn_star); + } + + combineStarfiles(fn_stars, MDout, obsModel); + +} + +void TomogramSet::convertBackFromSingleMetaDataTable(MetaDataTable &MDin, ObservationModel &obsModel) +{ + if (!MDin.containsLabel(EMDL_TOMO_TILT_MOVIE_INDEX)) + { + REPORT_ERROR("BUG: the MDin that is passed to TomogramSet::convertBackFromSingleMetaDataTable should contain a rlnTomoTiltMovieIndex label"); + } + + for (long int t = 0; t < globalTable.numberOfObjects(); t++) + { + tomogramObsModels[t] = obsModel; + MetaDataTable MDjoin = subsetMetaDataTable(MDin, EMDL_TOMO_NAME, tomogramNames[t], false); + + MDjoin.newSort(EMDL_TOMO_TILT_MOVIE_INDEX); + + if (MDjoin.numberOfObjects() != tomogramTables[t].numberOfObjects()) + { + REPORT_ERROR("ERROR: unequal number of Objects in tiltserie starfiles"); + } + + FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDjoin) + { + tomogramTables[t].setObject(MDjoin.getObject(), current_object); + } + + tomogramObsModels[t].removeUnusedOpticsGroups(tomogramTables[t]); + + } +} diff --git a/src/jaz/tomography/tomogram_set.h b/src/jaz/tomography/tomogram_set.h index d8d30c298..df91600ff 100644 --- a/src/jaz/tomography/tomogram_set.h +++ b/src/jaz/tomography/tomogram_set.h @@ -4,23 +4,29 @@ #include #include #include +#include +#include #include #include #include +#include "src/star_handling.h" class TomogramSet { public: - - TomogramSet(); - TomogramSet(std::string filename, bool verbose = true); - - - MetaDataTable globalTable; - std::vector tomogramTables; + MetaDataTable globalTable; + std::vector tomogramTables; + std::vector tomogramObsModels; + std::vector tomogramNames; + TomogramSet(); + TomogramSet(std::string filename, bool verbose = true); + + // return false if this is not a TomogramSet + bool read(std::string filename, bool verbose = true); + void write(FileName filename); Tomogram loadTomogram(int index, bool loadImageData) const; @@ -36,9 +42,7 @@ class TomogramSet const std::string& opticsGroupName); int size() const; - - void write(std::string filename) const; - + void setProjections(int tomogramIndex, const std::vector& proj); void setProjection(int tomogramIndex, int frame, const gravis::d4Matrix& P); void setCtf(int tomogramIndex, int frame, const CTF& ctf); @@ -62,6 +66,13 @@ class TomogramSet int getMaxFrameCount() const; double getPixelSize(int index) const; std::string getOpticsGroupName(int index) const; + + // Make one big metadatatable with all movies/micrographs (to be used for motioncorr and ctffind runners) + void generateSingleMetaDataTable(MetaDataTable &MDout, ObservationModel &obsModel); + + // Convert back from one big metadatatable into separate STAR files for each tilt serie + void convertBackFromSingleMetaDataTable(MetaDataTable &MDin, ObservationModel &obsModel); + }; #endif diff --git a/src/motioncorr_runner.cpp b/src/motioncorr_runner.cpp index 70ec15269..486692273 100644 --- a/src/motioncorr_runner.cpp +++ b/src/motioncorr_runner.cpp @@ -239,11 +239,11 @@ void MotioncorrRunner::initialise() MetaDataTable MDin; // Check if this is a TomographyExperiment starfile, and if so, unpack into one large metadatatable - if (tomo_model.read(fn_in, 1)) + if (tomogramSet.read(fn_in, 1)) { std::cerr <<"is_tomo"< +#include "src/jaz/tomography/tomogram_set.h" class MotioncorrRunner { @@ -70,7 +70,7 @@ class MotioncorrRunner bool is_tomo; // Information about tomography experiment - TomographyExperiment tomo_model; + TomogramSet tomogramSet; // Skip generation of logfile bool do_skip_logfile; diff --git a/src/pipeline_jobs.cpp b/src/pipeline_jobs.cpp index 2ce581551..01fdbbec6 100644 --- a/src/pipeline_jobs.cpp +++ b/src/pipeline_jobs.cpp @@ -1367,7 +1367,7 @@ void RelionJob::initialiseMotioncorrJob() if (is_tomo) { - joboptions["input_tilt_series"] = JobOption("Input tilt series set: ", OUTNODE_TOMO_TILT_SERIES, "", "Tilt series set STAR file (tilt_series.star)", "Input tomo tilt series set. Generated from CCPEM Import functionality..."); + joboptions["input_tomograms"] = JobOption("Input tilt series set: ", OUTNODE_TOMO_TOMOGRAMS, "", "Tomogram set STAR file (*.star)", "Input tomogram set."); } else { @@ -1432,13 +1432,13 @@ bool RelionJob::getCommandsMotioncorrJob(std::string &outputname, std::vector node_type2pipeliner_label = {{NODE_MOVIES_CPIPE, LABEL_MOVIES_CPIPE}, {NODE_MICS_CPIPE, LABEL_MICS_CPIPE}, @@ -508,8 +505,7 @@ static std::map node_type2pipeliner_label = {{NODE_MOVIES_CPIP {OUTNODE_TOMO_MAP, LABEL_TOMO_MAP}, {OUTNODE_TOMO_HALFMAP, LABEL_TOMO_HALFMAP}, {OUTNODE_TOMO_POST, LABEL_TOMO_POST}, - {OUTNODE_TOMO_POST_LOG, LABEL_TOMO_POST_LOG}, - {OUTNODE_TOMO_TILT_SERIES, LABEL_TOMO_TILT_SERIES}}; + {OUTNODE_TOMO_POST_LOG, LABEL_TOMO_POST_LOG}}; static std::map pipeliner_label2type = {{LABEL_MOVIES_CPIPE, NODE_MOVIES_CPIPE}, @@ -596,8 +592,7 @@ static std::map pipeliner_label2type = {{LABEL_MOVIES_CPIPE, N {LABEL_TOMO_MAP, OUTNODE_TOMO_MAP}, {LABEL_TOMO_HALFMAP, OUTNODE_TOMO_HALFMAP}, {LABEL_TOMO_POST, OUTNODE_TOMO_POST}, - {LABEL_TOMO_POST_LOG, OUTNODE_TOMO_POST_LOG}, - {LABEL_TOMO_TILT_SERIES, OUTNODE_TOMO_TILT_SERIES}}; + {LABEL_TOMO_POST_LOG, OUTNODE_TOMO_POST_LOG}}; //// Conversion dict for CCPEM-pipeliner compatibility static std::map node_label2type = {{NODE_MOVIES_LABEL, NODE_MOVIES}, diff --git a/src/tomo_model.cpp b/src/tomo_model.cpp deleted file mode 100644 index 34983156f..000000000 --- a/src/tomo_model.cpp +++ /dev/null @@ -1,189 +0,0 @@ -/*************************************************************************** - * - * Author: "Sjors H.W. Scheres" - * MRC Laboratory of Molecular Biology - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * This complete copyright notice must be included in any revised version of the - * source code. Additional authorship citations may be added, but existing - * author citations must be preserved. - ***************************************************************************/ - -#include "src/tomo_model.h" - - -RFLOAT ExpTiltSerie::getOpticsPixelSize(int optics_group) -{ - return obsModel.getPixelSize(optics_group); -} - -int ExpTiltSerie::getOpticsImageSize(int optics_group) -{ - return obsModel.getBoxSize(optics_group); -} - - - -bool TomographyExperiment::read(FileName fn_in, int verb) -{ - - clear(); - - if (!fn_in.isStarFile()) - { - if (verb > 0) std::cerr << " ERROR: input filename for TomographyExperiment is not a STAR file" << std::endl; - return false; - } - - MDtiltseries.read(fn_in, "tilt_series"); - if (MDtiltseries.numberOfObjects() == 0) - { - return false; - } - - if (!MDtiltseries.containsLabel(EMDL_TOMO_TILT_SERIES_STARFILE)) - { - REPORT_ERROR("ERROR: input starfile for TomographyExperiment " + fn_in + " does not contain rlnTomoTiltSeriesStarFile label "); - } - - if (!MDtiltseries.containsLabel(EMDL_TOMO_NAME)) - { - REPORT_ERROR("ERROR: input starfile for TomographyExperiment " + fn_in + " does not contain rlnTomoName label "); - } - - for (long int ts_id = 0; ts_id < MDtiltseries.numberOfObjects(); ts_id++) - { - ExpTiltSerie mytiltserie; - mytiltserie.id = ts_id; - - FileName fn_star; - MDtiltseries.getValue(EMDL_TOMO_TILT_SERIES_STARFILE, fn_star, ts_id); - MDtiltseries.getValue(EMDL_TOMO_NAME, mytiltserie.name, ts_id); - - ObservationModel::loadSafely(fn_star, mytiltserie.obsModel, mytiltserie.MDtiltimages, "tilt_images", verb); - if (mytiltserie.obsModel.opticsMdt.numberOfObjects() == 0) - { - REPORT_ERROR("ERROR: input starfile for tilt series " + fn_star + " does not contain any optics groups"); - } - if (!mytiltserie.MDtiltimages.containsLabel(EMDL_TOMO_TILT_MOVIE_INDEX)) - { - std::cerr << "table= "; mytiltserie.MDtiltimages.write(std::cerr); - REPORT_ERROR("ERROR: input starfile for tilt series " + fn_star + " does not contain rlnTomoTiltMovieIndex label "); - } - - // Make sure tilt images are sorted on their index (used to convert back from single large metadatatable) - mytiltserie.MDtiltimages.newSort(EMDL_TOMO_TILT_MOVIE_INDEX); - - /* - // Get all the necessary information about this tiltseries - for (long int timg_id = 0; timg_id < mytiltserie.MDtiltimages.numberOfObjects(); timg_id++) - { - ExpTiltImage mytiltimage; - mytiltimage.id = timg_id; - mytiltserie.MDtiltimages.getValue(EMDL_IMAGE_OPTICS_GROUP, mytiltimage.optics_group, timg_id); - mytiltimage.tiltseries_id = ts_id; - mytiltserie.tiltimages.push_back(mytiltimage); - } - */ - tiltseries.push_back(mytiltserie); - - } - - return true; - -} - - -void TomographyExperiment::write(FileName fn_outdir) -{ - // Make user fn_outdir ends in a slah: it should be a directory - if (fn_outdir[fn_outdir.size()-1]!='/') fn_outdir+='/'; - - // Write all the star files with the individual starfiles for the tilt images in the ouput directory - for (long int ts_id = 0; ts_id < MDtiltseries.numberOfObjects(); ts_id++) - { - - FileName fn_star; - MDtiltseries.getValue(EMDL_TOMO_TILT_SERIES_STARFILE, fn_star, ts_id); - FileName fn_newstar = getOutputFileWithNewUniqueDate(fn_star, fn_outdir); - MDtiltseries.setValue(EMDL_TOMO_TILT_SERIES_STARFILE, fn_newstar, ts_id); - - // Create output directory if neccesary - FileName newdir = fn_newstar.beforeLastOf("/"); - if (!exists(newdir)) - { - mktree(newdir); - } - - tiltseries[ts_id].obsModel.save(tiltseries[ts_id].MDtiltimages, fn_newstar, "tilt_images"); - - } - - // Also write the (now modified with fn_newstars) tilt_series.star file in the root directory - FileName fn_out = fn_outdir + "tilt_series.star"; - MDtiltseries.setName("tilt_series"); - MDtiltseries.write(fn_out); - -} - -long int TomographyExperiment::numberOfTiltImages() -{ - - long int result = 0; - for (long int i = 0; i < tiltseries.size(); i++) - { - result += tiltseries[i].numberOfTiltImages(); - } - return result; - -} - -void TomographyExperiment::generateSingleMetaDataTable(MetaDataTable &MDout, ObservationModel &obsModel) -{ - std::vector fn_stars; - for (long int ts_id = 0; ts_id < MDtiltseries.numberOfObjects(); ts_id++) - { - FileName fn_star; - MDtiltseries.getValue(EMDL_TOMO_TILT_SERIES_STARFILE, fn_star, ts_id); - fn_stars.push_back(fn_star); - } - - combineStarfiles(fn_stars, MDout, obsModel); - -} - -void TomographyExperiment::convertBackFromSingleMetaDataTable(MetaDataTable &MDin, ObservationModel &obsModel) -{ - if (!MDin.containsLabel(EMDL_TOMO_TILT_MOVIE_INDEX)) - { - REPORT_ERROR("BUG: the MDin that is passed to TomographyExperiment::convertBackFromSingleMetaDataTable should contain a rlnTomoTiltMovieIndex label"); - } - - for (long int ts_id = 0; ts_id < MDtiltseries.numberOfObjects(); ts_id++) - { - tiltseries[ts_id].obsModel = obsModel; - MetaDataTable MDjoin = subsetMetaDataTable(MDin, EMDL_TOMO_NAME, tiltseries[ts_id].name, false); - MDjoin.newSort(EMDL_TOMO_TILT_MOVIE_INDEX); - if (MDjoin.numberOfObjects() != tiltseries[ts_id].MDtiltimages.numberOfObjects()) - { - std::cerr << "MDjoin= "; MDjoin.write(std::cerr); - std::cerr << "tiltseries[ts_id].MDtiltimages= "; tiltseries[ts_id].MDtiltimages.write(std::cerr); - REPORT_ERROR("ERROR: unequal number of Objects in tiltserie starfiles"); - } - FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDjoin) - { - //MetaDataContainer* mine = MDjoin.getObject(); - tiltseries[ts_id].MDtiltimages.setObject(MDjoin.getObject(), current_object); - } - tiltseries[ts_id].obsModel.removeUnusedOpticsGroups(tiltseries[ts_id].MDtiltimages); - } -} diff --git a/src/tomo_model.h b/src/tomo_model.h deleted file mode 100644 index d3fa72925..000000000 --- a/src/tomo_model.h +++ /dev/null @@ -1,181 +0,0 @@ -/*************************************************************************** - * - * Author: "Sjors H.W. Scheres" - * MRC Laboratory of Molecular Biology - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * This complete copyright notice must be included in any revised version of the - * source code. Additional authorship citations may be added, but existing - * author citations must be preserved. - ***************************************************************************/ - -#ifndef RELION_TOMO_MODEL_H -#define RELION_TOMO_MODEL_H - -#include "src/metadata_table.h" -#include "src/star_handling.h" -#include - -/* -class ExpTiltImage -{ -public: - - // ID of this tiltimage inside its own tiltserie - long int id; - - // ID of the tiltserie that this tiltimage came from - long int tiltseries_id; - - // Optics group of the image - int optics_group; - - // Empty Constructor - ExpTiltImage() {} - - // Destructor needed for work with vectors - ~ExpTiltImage() {} - - // Copy constructor needed for work with vectors - ExpTiltImage(ExpTiltImage const& copy) - { - id = copy.id; - tiltseries_id = copy.tiltseries_id; - optics_group = copy.optics_group; - } - - // Define assignment operator in terms of the copy constructor - ExpTiltImage& operator=(ExpTiltImage const& copy) - { - id = copy.id; - tiltseries_id = copy.tiltseries_id; - optics_group = copy.optics_group; - return *this; - } - -}; -*/ - -class ExpTiltSerie -{ -public: - // All tiltimages in the tiltserie - //std::vector tiltimages; - - // ID of the tiltserie - long int id; - - // Name of the tiltserie (by this name it will be recognised upon reading) - std::string name; - - // Observation model holding the data for all optics groups - ObservationModel obsModel; - - // table with all the metadata - MetaDataTable MDtiltimages; - - // Empty Constructor - ExpTiltSerie() {} - - // Destructor needed for work with vectors - ~ExpTiltSerie() {} - - // Copy constructor needed for work with vectors - ExpTiltSerie(ExpTiltSerie const& copy) - { - //tiltimages = copy.tiltimages; - id = copy.id; - name = copy.name; - obsModel = copy.obsModel; - MDtiltimages = copy.MDtiltimages; - } - - // Define assignment operator in terms of the copy constructor - ExpTiltSerie& operator=(ExpTiltSerie const& copy) - { - //tiltimages = copy.tiltimages; - id = copy.id; - name = copy.name; - obsModel = copy.obsModel; - MDtiltimages = copy.MDtiltimages; - return *this; - } - long int numberOfTiltImages() - { - return MDtiltimages.numberOfObjects(); - } - - // Calculate the total number of optics groups in this tiltserie - int numberOfOpticsGroups() - { - return obsModel.numberOfOpticsGroups(); - } - - // Get the pixel size for this optics group - RFLOAT getOpticsPixelSize(int optics_group); - - // Get the original image size for this optics group - int getOpticsImageSize(int optics_group); - - -}; - - -class TomographyExperiment -{ -public: - // All tiltseries in the experiment - std::vector tiltseries; - - // Metadata table with information about all tiltseries - MetaDataTable MDtiltseries; - - // Empty Constructor - TomographyExperiment() - { - clear(); - } - - ~TomographyExperiment() - { - clear(); - } - - void clear() - { - tiltseries.clear(); - MDtiltseries.clear(); - } - - // Read from file, return false if this is not a TomographyExperiment - bool read(FileName fn_in, int verb = 0); - - // Write to file - void write(FileName fn_root); - - // Calculate the total number of tiltseries in this tomography experiment - long int numberOfTiltseries() - { - return tiltseries.size(); - } - - // Calculate the total number of tilt series images in this tomography experiment - long int numberOfTiltImages(); - - // Make one big metadatatable with all movies/micrographs (to be used for motioncorr and ctffind runners) - void generateSingleMetaDataTable(MetaDataTable &MDout, ObservationModel &obsModel); - - // Convert back from one big metadatatable into separate STAR files for each tilt serie - void convertBackFromSingleMetaDataTable(MetaDataTable &MDin, ObservationModel &obsModel); -}; - -#endif //RELION_TOMO_MODEL_H From edb500f5a3786168d38cf7a8fff241c5900acb56 Mon Sep 17 00:00:00 2001 From: scheres Date: Wed, 6 Apr 2022 20:15:59 +0100 Subject: [PATCH 009/495] saving a tomogram_set takes the file name, not the directory --- src/ctffind_runner.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ctffind_runner.cpp b/src/ctffind_runner.cpp index f1f835667..01bdb6da5 100644 --- a/src/ctffind_runner.cpp +++ b/src/ctffind_runner.cpp @@ -510,7 +510,7 @@ void CtffindRunner::joinCtffindResults() if (is_tomo) { tomogramSet.convertBackFromSingleMetaDataTable(MDctf, obsModel); - tomogramSet.write(fn_out); + tomogramSet.write(fn_out+"tomograms_ctf.star"); } else { From 25576ed77e2064739d9b4a0444d7b96e109a12f8 Mon Sep 17 00:00:00 2001 From: scheres Date: Thu, 7 Apr 2022 09:27:46 +0100 Subject: [PATCH 010/495] updated loadTomogram and addTomogram to work with the new obsModels --- src/jaz/tomography/tomogram_set.cpp | 45 +++++++++++++++++------------ src/jaz/tomography/tomogram_set.h | 4 +-- 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/src/jaz/tomography/tomogram_set.cpp b/src/jaz/tomography/tomogram_set.cpp index aa1e5254f..80e48f358 100644 --- a/src/jaz/tomography/tomogram_set.cpp +++ b/src/jaz/tomography/tomogram_set.cpp @@ -23,7 +23,6 @@ TomogramSet::TomogramSet(std::string filename, bool verbose) if (!read(filename, verbose)) { // This may be a tomograms.star file in the old, original relion-4.0 format. Try to convert - // TODO: make conversion from old tomograms.star to new tilt_series.star! std::ifstream ifs(filename); if (!ifs) @@ -190,10 +189,12 @@ Tomogram TomogramSet::loadTomogram(int index, bool loadImageData) const double Q0; - globalTable.getValueSafely(EMDL_TOMO_TILT_SERIES_PIXEL_SIZE, out.optics.pixelSize, index); - globalTable.getValueSafely(EMDL_CTF_VOLTAGE, out.optics.voltage, index); - globalTable.getValueSafely(EMDL_CTF_CS, out.optics.Cs, index); - globalTable.getValueSafely(EMDL_CTF_Q0, Q0, index); + // pixelsize, voltage, cs and q0 all need to be the same for all optics groups of one tomogram! + // Use values from the first optics group + tomogramObsModels[index].opticsMdt.getValueSafely(EMDL_TOMO_TILT_SERIES_PIXEL_SIZE, out.optics.pixelSize, 0); + tomogramObsModels[index].opticsMdt.getValueSafely(EMDL_CTF_VOLTAGE, out.optics.voltage, 0); + tomogramObsModels[index].opticsMdt.getValueSafely(EMDL_CTF_CS, out.optics.Cs, 0); + tomogramObsModels[index].opticsMdt.getValueSafely(EMDL_CTF_Q0, Q0, 0); out.hasDeformations = ( globalTable.containsLabel(EMDL_TOMO_DEFORMATION_GRID_SIZE_X) && @@ -299,6 +300,7 @@ Tomogram TomogramSet::loadTomogram(int index, bool loadImageData) const out.fractionalDose = out.cumulativeDose[out.frameSequence[1]] - out.cumulativeDose[out.frameSequence[0]]; } + // TODO: what to do about opticsGroups for a tomogram? This could be per frame now!!! if (globalTable.containsLabel(EMDL_IMAGE_OPTICS_GROUP_NAME)) { out.opticsGroupName = globalTable.getString(EMDL_IMAGE_OPTICS_GROUP_NAME, index); @@ -356,16 +358,22 @@ void TomogramSet::addTomogram( globalTable.setValue(EMDL_TOMO_SIZE_Y, h, index); globalTable.setValue(EMDL_TOMO_SIZE_Z, d, index); globalTable.setValue(EMDL_TOMO_HANDEDNESS, handedness, index); - - const CTF& ctf0 = ctfs[0]; - - globalTable.setValue(EMDL_IMAGE_OPTICS_GROUP_NAME, opticsGroupName, index); - globalTable.setValue(EMDL_TOMO_TILT_SERIES_PIXEL_SIZE, pixelSize, index); - globalTable.setValue(EMDL_CTF_VOLTAGE, ctf0.kV, index); - globalTable.setValue(EMDL_CTF_CS, ctf0.Cs, index); - globalTable.setValue(EMDL_CTF_Q0, ctf0.Q0, index); - globalTable.setValue(EMDL_TOMO_IMPORT_FRACT_DOSE, fractionalDose, index); - + globalTable.setValue(EMDL_TOMO_IMPORT_FRACT_DOSE, fractionalDose, index); + + const CTF& ctf0 = ctfs[0]; + + // TODO: how to handle multiple optics groups here? + // Anyway: this function is only called by relion_tomo_import_tomograms.cpp, which will disappear anyway... + MetaDataTable opticsMdt; + opticsMdt.addObject(); + opticsMdt.setValue(EMDL_IMAGE_OPTICS_GROUP_NAME, opticsGroupName); + opticsMdt.setValue(EMDL_TOMO_TILT_SERIES_PIXEL_SIZE, pixelSize); + opticsMdt.setValue(EMDL_CTF_VOLTAGE, ctf0.kV); + opticsMdt.setValue(EMDL_CTF_CS, ctf0.Cs); + opticsMdt.setValue(EMDL_CTF_Q0, ctf0.Q0); + ObservationModel obsModel(opticsMdt); + tomogramObsModels.push_back(obsModel); + if (tomogramTables.size() != index) { REPORT_ERROR_STR("TomogramSet::add: corrupted tomogram set: tomogramTables.size() = " @@ -546,18 +554,19 @@ int TomogramSet::getMaxFrameCount() const double TomogramSet::getPixelSize(int index) const { - return globalTable.getDouble(EMDL_TOMO_TILT_SERIES_PIXEL_SIZE, index); + // Get pixel size from the first optics group, as all optics group need the same pixel size + return tomogramObsModels[index].opticsMdt.getDouble(EMDL_TOMO_TILT_SERIES_PIXEL_SIZE, 0); } std::string TomogramSet::getOpticsGroupName(int index) const { - if (!globalTable.containsLabel(EMDL_IMAGE_OPTICS_GROUP_NAME)) + if (!tomogramObsModels[index].opticsMdt.containsLabel(EMDL_IMAGE_OPTICS_GROUP_NAME)) { return "opticsGroup1"; } else { - return globalTable.getString(EMDL_IMAGE_OPTICS_GROUP_NAME, index); + return tomogramObsModels[index].opticsMdt.getString(EMDL_IMAGE_OPTICS_GROUP_NAME, 0); } } diff --git a/src/jaz/tomography/tomogram_set.h b/src/jaz/tomography/tomogram_set.h index df91600ff..4b455cff9 100644 --- a/src/jaz/tomography/tomogram_set.h +++ b/src/jaz/tomography/tomogram_set.h @@ -67,10 +67,10 @@ class TomogramSet double getPixelSize(int index) const; std::string getOpticsGroupName(int index) const; - // Make one big metadatatable with all movies/micrographs (to be used for motioncorr and ctffind runners) + // SHWS 6Apr2022: Make one big metadatatable with all movies/micrographs (to be used for motioncorr and ctffind runners) void generateSingleMetaDataTable(MetaDataTable &MDout, ObservationModel &obsModel); - // Convert back from one big metadatatable into separate STAR files for each tilt serie + // SHWS 6Apr2022: Convert back from one big metadatatable into separate STAR files for each tilt serie void convertBackFromSingleMetaDataTable(MetaDataTable &MDin, ObservationModel &obsModel); }; From a8a3ae2ffb8317eb4b400c1a80ce7d04426307c0 Mon Sep 17 00:00:00 2001 From: scheres Date: Thu, 7 Apr 2022 14:48:35 +0100 Subject: [PATCH 011/495] moved back into Jasenkos tomogram_set, revoked star_handling etc --- src/apps/star_handler.cpp | 306 ++++++++++++++++- src/ctffind_runner.cpp | 15 +- src/ctffind_runner.h | 5 +- src/jaz/tomography/apps/import_tomograms.cpp | 7 +- src/jaz/tomography/tomogram_set.cpp | 161 ++++----- src/jaz/tomography/tomogram_set.h | 10 +- src/motioncorr_runner.cpp | 16 +- src/motioncorr_runner.h | 5 +- src/pipeline_jobs.cpp | 4 +- src/star_handling.cpp | 330 ------------------- src/star_handling.h | 36 -- 11 files changed, 400 insertions(+), 495 deletions(-) delete mode 100644 src/star_handling.cpp delete mode 100644 src/star_handling.h diff --git a/src/apps/star_handler.cpp b/src/apps/star_handler.cpp index e1b9b3276..3f8ababe1 100644 --- a/src/apps/star_handler.cpp +++ b/src/apps/star_handler.cpp @@ -20,7 +20,6 @@ #include #include -#include #include #include #include @@ -357,10 +356,307 @@ class star_handler_parameters if (fns_in.size() == 0) REPORT_ERROR("No STAR files to combine."); - MetaDataTable MDout; - combineStarfiles(fns_in, MDout, obsModel, fn_check, do_ignore_optics, tablename_in, 1); - - write_check_ignore_optics(MDout, fn_out, MDout.getName()); + MetaDataTable MDin0, MDout; + std::vector MDsin, MDoptics; + std::vector obsModels; + ObservationModel myobsModel0; + // Read the first table into the global obsModel + if (do_ignore_optics) MDin0.read(fns_in[0], tablename_in); + else ObservationModel::loadSafely(fns_in[0], obsModel, MDin0, "discover", 1); + MDsin.push_back(MDin0); + // Read all the rest of the tables into local obsModels + for (int i = 1; i < fns_in.size(); i++) + { + ObservationModel myobsModel; + MetaDataTable MDin; // define again, as reading from previous one may linger here... + if (do_ignore_optics) MDin.read(fns_in[i], tablename_in); + else ObservationModel::loadSafely(fns_in[i], myobsModel, MDin, "discover", 1); + MDsin.push_back(MDin); + obsModels.push_back(myobsModel); + } + + // Combine optics groups with the same EMDL_IMAGE_OPTICS_GROUP_NAME, make new ones for those with a different name + if (!do_ignore_optics) + { + std::vector optics_group_uniq_names; + + // Initialise optics_group_uniq_names with the first table + FOR_ALL_OBJECTS_IN_METADATA_TABLE(obsModel.opticsMdt) + { + std::string myname; + obsModel.opticsMdt.getValue(EMDL_IMAGE_OPTICS_GROUP_NAME, myname); + optics_group_uniq_names.push_back(myname); + } + + // Now check uniqueness of the other tables + for (int MDs_id = 1; MDs_id < fns_in.size(); MDs_id++) + { + const int obs_id = MDs_id - 1; + + std::vector new_optics_groups; + FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDsin[MDs_id]) + { + int tmp; + MDsin[MDs_id].getValue(EMDL_IMAGE_OPTICS_GROUP, tmp); + new_optics_groups.push_back(tmp); + } + + MetaDataTable unique_opticsMdt; + unique_opticsMdt.addMissingLabels(&obsModels[obs_id].opticsMdt); + + FOR_ALL_OBJECTS_IN_METADATA_TABLE(obsModels[obs_id].opticsMdt) + { + std::string myname; + int my_optics_group; + obsModels[obs_id].opticsMdt.getValue(EMDL_IMAGE_OPTICS_GROUP_NAME, myname); + obsModels[obs_id].opticsMdt.getValue(EMDL_IMAGE_OPTICS_GROUP, my_optics_group); + + // Check whether this name is unique + bool is_uniq = true; + int new_group; + for (new_group = 0; new_group < optics_group_uniq_names.size(); new_group++) + { + if (optics_group_uniq_names[new_group] == myname) + { + is_uniq = false; + break; + } + } + new_group ++; // start counting of groups at 1, not 0! + + if (is_uniq) + { + std::cout << " + Adding new optics_group with name: " << myname << std::endl; + + optics_group_uniq_names.push_back(myname); + // Add the line to the global obsModel + obsModels[obs_id].opticsMdt.setValue(EMDL_IMAGE_OPTICS_GROUP, new_group); + + unique_opticsMdt.addObject(); + unique_opticsMdt.setObject(obsModels[obs_id].opticsMdt.getObject()); + } + else + { + std::cout << " + Joining optics_groups with the same name: " << myname << std::endl; + std::cerr << " + WARNING: if these are different data sets, you might want to rename optics groups instead of joining them!" << std::endl; + std::cerr << " + WARNING: if so, manually edit the rlnOpticsGroupName column in the optics_groups table of your input STAR files." << std::endl; + } + + if (my_optics_group != new_group) + { + std::cout << " + Renumbering group " << myname << " from " << my_optics_group << " to " << new_group << std::endl; + } + + // Update the optics_group entry for all particles in the MDsin + for (long int current_object2 = MDsin[MDs_id].firstObject(); + current_object2 < MDsin[MDs_id].numberOfObjects() && current_object2 >= 0; + current_object2 = MDsin[MDs_id].nextObject()) + { + int old_optics_group; + MDsin[MDs_id].getValue(EMDL_IMAGE_OPTICS_GROUP, old_optics_group, current_object2); + if (old_optics_group == my_optics_group) + new_optics_groups[current_object2] = new_group; + } + } + + obsModels[obs_id].opticsMdt = unique_opticsMdt; + + FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDsin[MDs_id]) + { + MDsin[MDs_id].setValue(EMDL_IMAGE_OPTICS_GROUP, new_optics_groups[current_object]); + + // Also rename the rlnGroupName to not have groups overlapping from different optics groups + std::string name; + if (MDsin[MDs_id].getValue(EMDL_MLMODEL_GROUP_NAME, name)) + { + name = "optics"+integerToString(new_optics_groups[current_object])+"_"+name; + MDsin[MDs_id].setValue(EMDL_MLMODEL_GROUP_NAME, name); + } + } + } + + // Make one vector for combination of the optics tables + MDoptics.push_back(obsModel.opticsMdt); + for (int i = 1; i < fns_in.size(); i++) + { + MDoptics.push_back(obsModels[i - 1].opticsMdt); + } + + // Check if anisotropic magnification and/or beam_tilt are present in some optics groups, but not in others. + // If so, initialise the others correctly + bool has_beamtilt = false, has_not_beamtilt = false; + bool has_anisomag = false, has_not_anisomag = false; + bool has_odd_zernike = false, has_not_odd_zernike = false; + bool has_even_zernike = false, has_not_even_zernike = false; + bool has_ctf_premultiplied = false, has_not_ctf_premultiplied = false; + for (int i = 0; i < fns_in.size(); i++) + { + if (MDoptics[i].containsLabel(EMDL_IMAGE_BEAMTILT_X) || + MDoptics[i].containsLabel(EMDL_IMAGE_BEAMTILT_Y)) + { + has_beamtilt = true; + } + else + { + has_not_beamtilt = true; + } + if (MDoptics[i].containsLabel(EMDL_IMAGE_MAG_MATRIX_00) && + MDoptics[i].containsLabel(EMDL_IMAGE_MAG_MATRIX_01) && + MDoptics[i].containsLabel(EMDL_IMAGE_MAG_MATRIX_10) && + MDoptics[i].containsLabel(EMDL_IMAGE_MAG_MATRIX_11)) + { + has_anisomag = true; + } + else + { + has_not_anisomag = true; + } + if (MDoptics[i].containsLabel(EMDL_IMAGE_ODD_ZERNIKE_COEFFS)) + { + has_odd_zernike = true; + } + else + { + has_not_odd_zernike = true; + } + if (MDoptics[i].containsLabel(EMDL_IMAGE_EVEN_ZERNIKE_COEFFS)) + { + has_even_zernike = true; + } + else + { + has_not_even_zernike = true; + } + if (MDoptics[i].containsLabel(EMDL_OPTIMISER_DATA_ARE_CTF_PREMULTIPLIED)) + { + has_ctf_premultiplied = true; + } + else + { + has_not_ctf_premultiplied = true; + } + } +#ifdef DEBUG + printf("has_beamtilt = %d, has_not_beamtilt = %d, has_anisomag = %d, has_not_anisomag = %d, has_odd_zernike = %d, has_not_odd_zernike = %d, has_even_zernike = %d, has_not_even_zernike = %d, has_ctf_premultiplied = %d, has_not_ctf_premultiplied = %d\n", has_beamtilt, has_not_beamtilt, has_anisomag, has_not_anisomag, has_odd_zernike, has_not_odd_zernike, has_even_zernike, has_not_even_zernike, has_ctf_premultiplied, has_not_ctf_premultiplied); +#endif + + for (int i = 0; i < fns_in.size(); i++) + { + if (has_beamtilt && has_not_beamtilt) + { + if (!MDoptics[i].containsLabel(EMDL_IMAGE_BEAMTILT_X)) + { + FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDoptics[i]) + { + MDoptics[i].setValue(EMDL_IMAGE_BEAMTILT_X, 0.); + } + } + if (!MDoptics[i].containsLabel(EMDL_IMAGE_BEAMTILT_Y)) + { + FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDoptics[i]) + { + MDoptics[i].setValue(EMDL_IMAGE_BEAMTILT_Y, 0.); + } + } + } + + if (has_anisomag && has_not_anisomag) + { + if (!(MDoptics[i].containsLabel(EMDL_IMAGE_MAG_MATRIX_00) && + MDoptics[i].containsLabel(EMDL_IMAGE_MAG_MATRIX_01) && + MDoptics[i].containsLabel(EMDL_IMAGE_MAG_MATRIX_10) && + MDoptics[i].containsLabel(EMDL_IMAGE_MAG_MATRIX_11)) ) + { + FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDoptics[i]) + { + MDoptics[i].setValue(EMDL_IMAGE_MAG_MATRIX_00, 1.); + MDoptics[i].setValue(EMDL_IMAGE_MAG_MATRIX_01, 0.); + MDoptics[i].setValue(EMDL_IMAGE_MAG_MATRIX_10, 0.); + MDoptics[i].setValue(EMDL_IMAGE_MAG_MATRIX_11, 1.); + } + } + } + + if (has_odd_zernike && has_not_odd_zernike) + { + std::vector six_zeros(6, 0); + if (!MDoptics[i].containsLabel(EMDL_IMAGE_ODD_ZERNIKE_COEFFS)) + { + FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDoptics[i]) + { + MDoptics[i].setValue(EMDL_IMAGE_ODD_ZERNIKE_COEFFS, six_zeros); + } + } + } + + if (has_even_zernike && has_not_even_zernike) + { + std::vector nine_zeros(9, 0); + if (!MDoptics[i].containsLabel(EMDL_IMAGE_EVEN_ZERNIKE_COEFFS)) + { + FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDoptics[i]) + { + MDoptics[i].setValue(EMDL_IMAGE_EVEN_ZERNIKE_COEFFS, nine_zeros); + } + } + } + + if (has_ctf_premultiplied && has_not_ctf_premultiplied) + { + if (!MDoptics[i].containsLabel(EMDL_OPTIMISER_DATA_ARE_CTF_PREMULTIPLIED)) + { + FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDoptics[i]) + { + MDoptics[i].setValue(EMDL_OPTIMISER_DATA_ARE_CTF_PREMULTIPLIED, false); + } + } + } + } + + // Now combine all optics tables into one + obsModel.opticsMdt = MetaDataTable::combineMetaDataTables(MDoptics); + } + + // Combine the particles tables + MDout = MetaDataTable::combineMetaDataTables(MDsin); + + //Deactivate the group_name column + MDout.deactivateLabel(EMDL_MLMODEL_GROUP_NO); + + if (fn_check != "") + { + EMDLabel label = EMDL::str2Label(fn_check); + if (!MDout.containsLabel(label)) + REPORT_ERROR("ERROR: the output file does not contain the label to check for duplicates. Is it present in all input files?"); + + /// Don't want to mess up original order, so make a MDsort with only that label... + FileName fn_this, fn_prev = ""; + MetaDataTable MDsort; + FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDout) + { + MDout.getValue(label, fn_this); + MDsort.addObject(); + MDsort.setValue(label, fn_this); + } + // sort on the label + MDsort.newSort(label); + long int nr_duplicates = 0; + FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDsort) + { + MDsort.getValue(label, fn_this); + if (fn_this == fn_prev) + { + nr_duplicates++; + std::cerr << " WARNING: duplicate entry: " << fn_this << std::endl; + } + fn_prev = fn_this; + } + + if (nr_duplicates > 0) + std::cerr << " WARNING: Total number of duplicate "<< fn_check << " entries: " << nr_duplicates << std::endl; + } + + write_check_ignore_optics(MDout, fn_out, MDin0.getName()); std::cout << " Written: " << fn_out << std::endl; } diff --git a/src/ctffind_runner.cpp b/src/ctffind_runner.cpp index 01bdb6da5..8771a7b05 100644 --- a/src/ctffind_runner.cpp +++ b/src/ctffind_runner.cpp @@ -191,12 +191,9 @@ void CtffindRunner::initialise(bool is_leader) if (is_tomo) { - FileName fn_tomo; - MDin.getValue(EMDL_TOMO_NAME, fn_tomo); - fn_tomogram_names.push_back(fn_tomo); - long index; - MDin.getValue(EMDL_TOMO_TILT_MOVIE_INDEX, index); - tomo_tilt_movie_index.push_back(index); + RFLOAT exposure; + MDin.getValue(EMDL_MICROGRAPH_PRE_EXPOSURE, exposure); + pre_exposure_micrographs.push_back(exposure); } } @@ -498,8 +495,8 @@ void CtffindRunner::joinCtffindResults() if (is_tomo) { - MDctf.setValue(EMDL_TOMO_NAME, fn_tomogram_names[imic]); - MDctf.setValue(EMDL_TOMO_TILT_MOVIE_INDEX, tomo_tilt_movie_index[imic]); + // Store pre-exposure to sort images on, just in case this program messed up the order... + MDctf.setValue(EMDL_MICROGRAPH_PRE_EXPOSURE, pre_exposure_micrographs[imic]); } } @@ -510,7 +507,7 @@ void CtffindRunner::joinCtffindResults() if (is_tomo) { tomogramSet.convertBackFromSingleMetaDataTable(MDctf, obsModel); - tomogramSet.write(fn_out+"tomograms_ctf.star"); + tomogramSet.write(fn_out+"tilt_series_ctf.star"); } else { diff --git a/src/ctffind_runner.h b/src/ctffind_runner.h index ab73ade18..577c5ad20 100644 --- a/src/ctffind_runner.h +++ b/src/ctffind_runner.h @@ -59,10 +59,7 @@ class CtffindRunner ObservationModel obsModel; // Tilt movie index for each micrograph in a tilt serie (needed for converting back to tomographyExperiment) - std::vector tomo_tilt_movie_index; - - // Tomogram names of all the micrographs - std::vector fn_tomogram_names; + std::vector pre_exposure_micrographs; // Is this a tomography experiment? bool is_tomo; diff --git a/src/jaz/tomography/apps/import_tomograms.cpp b/src/jaz/tomography/apps/import_tomograms.cpp index 8ad2fb8bc..acd5b4333 100644 --- a/src/jaz/tomography/apps/import_tomograms.cpp +++ b/src/jaz/tomography/apps/import_tomograms.cpp @@ -381,17 +381,12 @@ int main(int argc, char *argv[]) ctfs[i] = ctfs[mapping.oldFrameIndex[i]]; } - std::string opticsGroupName = "opticsGroup1"; - - perTomoArguments.getValue(EMDL_IMAGE_OPTICS_GROUP_NAME, opticsGroupName, tomo_index); - - tomograms.addTomogram( name, tsFn, mapping.projections, mapping.w, mapping.h, mapping.d, cumulativeDose, fractionalDose, - ctfs, hand, pixelSize, opticsGroupName); + ctfs, hand, pixelSize); Log::endSection(); } diff --git a/src/jaz/tomography/tomogram_set.cpp b/src/jaz/tomography/tomogram_set.cpp index 80e48f358..8b6865ec5 100644 --- a/src/jaz/tomography/tomogram_set.cpp +++ b/src/jaz/tomography/tomogram_set.cpp @@ -18,11 +18,12 @@ TomogramSet::TomogramSet() globalTable.setName("global"); } -TomogramSet::TomogramSet(std::string filename, bool verbose) +TomogramSet::TomogramSet(FileName filename, bool verbose) { if (!read(filename, verbose)) { // This may be a tomograms.star file in the old, original relion-4.0 format. Try to convert + FileName mydir = filename.beforeLastOf("/"); std::ifstream ifs(filename); if (!ifs) @@ -32,32 +33,22 @@ TomogramSet::TomogramSet(std::string filename, bool verbose) else { globalTable.readStar(ifs, "global"); + globalTable.setName("global"); if (!globalTable.containsLabel(EMDL_TOMO_NAME)) { REPORT_ERROR("ERROR: input starfile for TomogramSet " + filename + " does not contain rlnTomoName label "); } - // remove information from optics groups and move into globalObsModel - ObservationModel globalObsModel(globalTable); - globalTable.deactivateLabel(EMDL_IMAGE_OPTICS_GROUP_NAME); - globalTable.deactivateLabel(EMDL_IMAGE_OPTICS_GROUP); - globalTable.deactivateLabel(EMDL_CTF_CS); - globalTable.deactivateLabel(EMDL_CTF_VOLTAGE); - globalTable.deactivateLabel(EMDL_CTF_Q0); - const int tc = globalTable.numberOfObjects(); tomogramTables.resize(tc); - tomogramObsModels.resize(tc); - tomogramNames.resize(tc); std::vector allTables = MetaDataTable::readAll(ifs, tc+1); - for (int t = 0; t < tc; t++) { - const std::string expectedNewName = globalTable.getString(EMDL_TOMO_NAME, t); - const std::string name = allTables[t+1].getName(); + FileName expectedNewName = globalTable.getString(EMDL_TOMO_NAME, t); + FileName name = allTables[t+1].getName(); if (name != expectedNewName) { @@ -65,22 +56,37 @@ TomogramSet::TomogramSet(std::string filename, bool verbose) } tomogramTables[t] = allTables[t+1]; - tomogramTables[t].setName("tilt_images"); - tomogramObsModels[t] = globalObsModel; - tomogramNames[t] = expectedNewName; + tomogramTables[t].setName(expectedNewName); + + // Check there is a rlnTomoTiltMovieIndex label, otherwise add one + if (!tomogramTables[t].containsLabel(EMDL_MICROGRAPH_PRE_EXPOSURE) ) + { + REPORT_ERROR("ERROR: tilt series " + expectedNewName + " does not contain compulsory rlnMicrographPreExposure label"); + } + + // As this is a conversion, also save already all the tilt series starfiles in a new directory + // Create output directory if necessary + FileName fn_star = mydir + "/tilt_series/" + expectedNewName + ".star"; + globalTable.setValue(EMDL_TOMO_TILT_SERIES_STARFILE, fn_star, t); + FileName newdir = fn_star.beforeLastOf("/"); + if (!exists(newdir)) mktree(newdir); + + // Write the individual tomogram starfile + tomogramTables[t].newSort(EMDL_MICROGRAPH_PRE_EXPOSURE); + tomogramTables[t].write(fn_star); + } } } - globalTable.setName("global"); - } bool TomogramSet::read(std::string filename, bool verbose) { globalTable.read(filename, "global"); + globalTable.setName("global"); const int tc = globalTable.numberOfObjects(); @@ -100,16 +106,17 @@ bool TomogramSet::read(std::string filename, bool verbose) } tomogramTables.resize(tc); - tomogramObsModels.resize(tc); - tomogramNames.resize(tc); for (int t = 0; t < tc; t++) { - tomogramNames[t] = globalTable.getString(EMDL_TOMO_NAME, t); + FileName name = globalTable.getString(EMDL_TOMO_NAME, t); std::string fn_star = globalTable.getString(EMDL_TOMO_TILT_SERIES_STARFILE, t); - ObservationModel::loadSafely(fn_star, tomogramObsModels[t], tomogramTables[t], "tilt_images", verbose); - // Make sure tilt images are sorted on their index (used to convert back from single large metadatatable) - tomogramTables[t].newSort(EMDL_TOMO_TILT_MOVIE_INDEX); + tomogramTables[t].read(fn_star, name); + // Make sure tilt images are sorted on their pre-exposure (used to convert back from single large metadatatable) + if (tomogramTables[t].containsLabel(EMDL_MICROGRAPH_PRE_EXPOSURE)) + tomogramTables[t].newSort(EMDL_MICROGRAPH_PRE_EXPOSURE); + else + REPORT_ERROR("ERROR: tomogramTable does not contain compulsory rlnMicrographPreExposure label"); } return true; @@ -135,7 +142,7 @@ void TomogramSet::write(FileName filename) if (!exists(newdir)) mktree(newdir); // Write the individual tomogram starfile - tomogramObsModels[t].save(tomogramTables[t], fn_newstar, "tilt_images"); + tomogramTables[t].write(fn_newstar); } // Also write the (now modified with fn_newstars) tilt_series.star file in the root directory @@ -189,12 +196,10 @@ Tomogram TomogramSet::loadTomogram(int index, bool loadImageData) const double Q0; - // pixelsize, voltage, cs and q0 all need to be the same for all optics groups of one tomogram! - // Use values from the first optics group - tomogramObsModels[index].opticsMdt.getValueSafely(EMDL_TOMO_TILT_SERIES_PIXEL_SIZE, out.optics.pixelSize, 0); - tomogramObsModels[index].opticsMdt.getValueSafely(EMDL_CTF_VOLTAGE, out.optics.voltage, 0); - tomogramObsModels[index].opticsMdt.getValueSafely(EMDL_CTF_CS, out.optics.Cs, 0); - tomogramObsModels[index].opticsMdt.getValueSafely(EMDL_CTF_Q0, Q0, 0); + globalTable.getValueSafely(EMDL_MICROGRAPH_ORIGINAL_PIXEL_SIZE, out.optics.pixelSize, index); + globalTable.getValueSafely(EMDL_CTF_VOLTAGE, out.optics.voltage, index); + globalTable.getValueSafely(EMDL_CTF_CS, out.optics.Cs, index); + globalTable.getValueSafely(EMDL_CTF_Q0, Q0, index); out.hasDeformations = ( globalTable.containsLabel(EMDL_TOMO_DEFORMATION_GRID_SIZE_X) && @@ -300,17 +305,6 @@ Tomogram TomogramSet::loadTomogram(int index, bool loadImageData) const out.fractionalDose = out.cumulativeDose[out.frameSequence[1]] - out.cumulativeDose[out.frameSequence[0]]; } - // TODO: what to do about opticsGroups for a tomogram? This could be per frame now!!! - if (globalTable.containsLabel(EMDL_IMAGE_OPTICS_GROUP_NAME)) - { - out.opticsGroupName = globalTable.getString(EMDL_IMAGE_OPTICS_GROUP_NAME, index); - } - else - { - out.opticsGroupName = "opticsGroup1"; - } - - out.name = tomoName; if (globalTable.containsLabel(EMDL_TOMO_FIDUCIALS_STARFILE)) @@ -342,8 +336,7 @@ void TomogramSet::addTomogram( double fractionalDose, const std::vector& ctfs, double handedness, - double pixelSize, - const std::string& opticsGroupName) + double pixelSize) { const int index = globalTable.numberOfObjects(); const int fc = projections.size(); @@ -358,21 +351,15 @@ void TomogramSet::addTomogram( globalTable.setValue(EMDL_TOMO_SIZE_Y, h, index); globalTable.setValue(EMDL_TOMO_SIZE_Z, d, index); globalTable.setValue(EMDL_TOMO_HANDEDNESS, handedness, index); - globalTable.setValue(EMDL_TOMO_IMPORT_FRACT_DOSE, fractionalDose, index); const CTF& ctf0 = ctfs[0]; - // TODO: how to handle multiple optics groups here? - // Anyway: this function is only called by relion_tomo_import_tomograms.cpp, which will disappear anyway... - MetaDataTable opticsMdt; - opticsMdt.addObject(); - opticsMdt.setValue(EMDL_IMAGE_OPTICS_GROUP_NAME, opticsGroupName); - opticsMdt.setValue(EMDL_TOMO_TILT_SERIES_PIXEL_SIZE, pixelSize); - opticsMdt.setValue(EMDL_CTF_VOLTAGE, ctf0.kV); - opticsMdt.setValue(EMDL_CTF_CS, ctf0.Cs); - opticsMdt.setValue(EMDL_CTF_Q0, ctf0.Q0); - ObservationModel obsModel(opticsMdt); - tomogramObsModels.push_back(obsModel); + globalTable.setValue(EMDL_MICROGRAPH_ORIGINAL_PIXEL_SIZE, pixelSize, index); + globalTable.setValue(EMDL_CTF_VOLTAGE, ctf0.kV, index); + globalTable.setValue(EMDL_CTF_CS, ctf0.Cs, index); + globalTable.setValue(EMDL_CTF_Q0, ctf0.Q0, index); + globalTable.setValue(EMDL_TOMO_IMPORT_FRACT_DOSE, fractionalDose, index); + if (tomogramTables.size() != index) { @@ -554,60 +541,78 @@ int TomogramSet::getMaxFrameCount() const double TomogramSet::getPixelSize(int index) const { - // Get pixel size from the first optics group, as all optics group need the same pixel size - return tomogramObsModels[index].opticsMdt.getDouble(EMDL_TOMO_TILT_SERIES_PIXEL_SIZE, 0); + return globalTable.getDouble(EMDL_MICROGRAPH_ORIGINAL_PIXEL_SIZE, index); } std::string TomogramSet::getOpticsGroupName(int index) const { - if (!tomogramObsModels[index].opticsMdt.containsLabel(EMDL_IMAGE_OPTICS_GROUP_NAME)) + if (!globalTable.containsLabel(EMDL_IMAGE_OPTICS_GROUP_NAME)) { return "opticsGroup1"; } else { - return tomogramObsModels[index].opticsMdt.getString(EMDL_IMAGE_OPTICS_GROUP_NAME, 0); + return globalTable.getString(EMDL_IMAGE_OPTICS_GROUP_NAME, index); } } void TomogramSet::generateSingleMetaDataTable(MetaDataTable &MDout, ObservationModel &obsModel) { - std::vector fn_stars; - for (long int t = 0; t < globalTable.numberOfObjects(); t++) + MDout.clear(); + for (long int t = 0; t < tomogramTables.size(); t++) { - FileName fn_star = globalTable.getString(EMDL_TOMO_TILT_SERIES_STARFILE, t); - fn_stars.push_back(fn_star); - } - - combineStarfiles(fn_stars, MDout, obsModel); + // Store all the necessary optics stuff in an opticsGroup per tomogram + RFLOAT pixelSize, voltage, Cs, Q0; + std::string tomo_name = getTomogramName(t); + globalTable.getValueSafely(EMDL_MICROGRAPH_ORIGINAL_PIXEL_SIZE, pixelSize, t); + globalTable.getValueSafely(EMDL_CTF_VOLTAGE, voltage, t); + globalTable.getValueSafely(EMDL_CTF_CS, Cs, t); + globalTable.getValueSafely(EMDL_CTF_Q0, Q0, t); + obsModel.opticsMdt.addObject(); + obsModel.opticsMdt.setValue(EMDL_IMAGE_OPTICS_GROUP_NAME, tomo_name); + obsModel.opticsMdt.setValue(EMDL_IMAGE_OPTICS_GROUP, t+1); + obsModel.opticsMdt.setValue(EMDL_MICROGRAPH_ORIGINAL_PIXEL_SIZE, pixelSize); + obsModel.opticsMdt.setValue(EMDL_CTF_VOLTAGE, voltage); + obsModel.opticsMdt.setValue(EMDL_CTF_CS, Cs); + obsModel.opticsMdt.setValue(EMDL_CTF_Q0, Q0); + + FOR_ALL_OBJECTS_IN_METADATA_TABLE(tomogramTables[t]) + { + tomogramTables[t].setValue(EMDL_IMAGE_OPTICS_GROUP, t+1); + } + MDout.append(tomogramTables[t]); + } } void TomogramSet::convertBackFromSingleMetaDataTable(MetaDataTable &MDin, ObservationModel &obsModel) { - if (!MDin.containsLabel(EMDL_TOMO_TILT_MOVIE_INDEX)) + if (!MDin.containsLabel(EMDL_MICROGRAPH_PRE_EXPOSURE)) { - REPORT_ERROR("BUG: the MDin that is passed to TomogramSet::convertBackFromSingleMetaDataTable should contain a rlnTomoTiltMovieIndex label"); + REPORT_ERROR("BUG: MDin should contain a rlnMicrographPreExposure label"); } - for (long int t = 0; t < globalTable.numberOfObjects(); t++) + for (long int t = 0; t < tomogramTables.size(); t++) { - tomogramObsModels[t] = obsModel; - MetaDataTable MDjoin = subsetMetaDataTable(MDin, EMDL_TOMO_NAME, tomogramNames[t], false); + MetaDataTable MDsub = subsetMetaDataTable(MDin, EMDL_IMAGE_OPTICS_GROUP, t+1, t+1); - MDjoin.newSort(EMDL_TOMO_TILT_MOVIE_INDEX); + // Make sure no one unsorted the tilt images in each serie... + if (MDsub.containsLabel(EMDL_MICROGRAPH_PRE_EXPOSURE)) + MDsub.newSort(EMDL_MICROGRAPH_PRE_EXPOSURE); + else + REPORT_ERROR("BUG: MDsub does no longer contain a rlnMicrographPreExposure label"); - if (MDjoin.numberOfObjects() != tomogramTables[t].numberOfObjects()) + if (MDsub.numberOfObjects() != tomogramTables[t].numberOfObjects()) { REPORT_ERROR("ERROR: unequal number of Objects in tiltserie starfiles"); } - FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDjoin) + FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDsub) { - tomogramTables[t].setObject(MDjoin.getObject(), current_object); + tomogramTables[t].setObject(MDsub.getObject(), current_object); } - tomogramObsModels[t].removeUnusedOpticsGroups(tomogramTables[t]); + tomogramTables[t].deactivateLabel(EMDL_IMAGE_OPTICS_GROUP); } } diff --git a/src/jaz/tomography/tomogram_set.h b/src/jaz/tomography/tomogram_set.h index 4b455cff9..46a44726d 100644 --- a/src/jaz/tomography/tomogram_set.h +++ b/src/jaz/tomography/tomogram_set.h @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include @@ -18,11 +17,9 @@ class TomogramSet MetaDataTable globalTable; std::vector tomogramTables; - std::vector tomogramObsModels; - std::vector tomogramNames; TomogramSet(); - TomogramSet(std::string filename, bool verbose = true); + TomogramSet(FileName filename, bool verbose = true); // return false if this is not a TomogramSet bool read(std::string filename, bool verbose = true); @@ -38,8 +35,7 @@ class TomogramSet double fractionalDose, const std::vector& ctfs, double handedness, - double pixelSize, - const std::string& opticsGroupName); + double pixelSize); int size() const; @@ -60,7 +56,7 @@ class TomogramSet void clearDeformation(); int getTomogramIndex(std::string tomogramName) const; - std::string getTomogramName(int index) const; + std::string getTomogramName(int index) const; int getTomogramIndexSafely(std::string tomogramName) const; int getFrameCount(int index) const; int getMaxFrameCount() const; diff --git a/src/motioncorr_runner.cpp b/src/motioncorr_runner.cpp index 486692273..4c57b9a5e 100644 --- a/src/motioncorr_runner.cpp +++ b/src/motioncorr_runner.cpp @@ -241,7 +241,6 @@ void MotioncorrRunner::initialise() // Check if this is a TomographyExperiment starfile, and if so, unpack into one large metadatatable if (tomogramSet.read(fn_in, 1)) { - std::cerr <<"is_tomo"< fn_micrographs, fn_ori_micrographs, fn_tomogram_names; + std::vector fn_micrographs, fn_ori_micrographs; // Optics group number for all original micrographs std::vector optics_group_micrographs, optics_group_ori_micrographs; @@ -60,9 +60,6 @@ class MotioncorrRunner // Pre-exposure for each micrograph (mainly used for tomography) std::vector pre_exposure_micrographs; - // Tilt movie index for each micrograph in a tilt serie (needed for converting back to tomographyExperiment) - std::vector tomo_tilt_movie_index; - // Information about the optics groups ObservationModel obsModel; diff --git a/src/pipeline_jobs.cpp b/src/pipeline_jobs.cpp index 01fdbbec6..7369a3084 100644 --- a/src/pipeline_jobs.cpp +++ b/src/pipeline_jobs.cpp @@ -1457,7 +1457,7 @@ bool RelionJob::getCommandsMotioncorrJob(std::string &outputname, std::vector &fns_in, - MetaDataTable &MDout, ObservationModel &obsModel, - FileName fn_check, bool do_ignore_optics, std::string tablename_in, int verb) -{ - MetaDataTable MDin0; - std::vector MDsin, MDoptics; - std::vector obsModels; - ObservationModel myobsModel0; - - // Read the first table into the global obsModel - if (do_ignore_optics) MDin0.read(fns_in[0], tablename_in); - else ObservationModel::loadSafely(fns_in[0], obsModel, MDin0, "discover", 1); - MDsin.push_back(MDin0); - - // Read all the rest of the tables into local obsModels - for (int i = 1; i < fns_in.size(); i++) - { - ObservationModel myobsModel; - MetaDataTable MDin; // define again, as reading from previous one may linger here... - if (do_ignore_optics) MDin.read(fns_in[i], tablename_in); - else ObservationModel::loadSafely(fns_in[i], myobsModel, MDin, "discover", 1); - MDsin.push_back(MDin); - obsModels.push_back(myobsModel); - } - - // Combine optics groups with the same EMDL_IMAGE_OPTICS_GROUP_NAME, make new ones for those with a different name - if (!do_ignore_optics) - { - std::vector optics_group_uniq_names; - - // Initialise optics_group_uniq_names with the first table - FOR_ALL_OBJECTS_IN_METADATA_TABLE(obsModel.opticsMdt) - { - std::string myname; - obsModel.opticsMdt.getValue(EMDL_IMAGE_OPTICS_GROUP_NAME, myname); - optics_group_uniq_names.push_back(myname); - } - - // Now check uniqueness of the other tables - for (int MDs_id = 1; MDs_id < fns_in.size(); MDs_id++) - { - const int obs_id = MDs_id - 1; - - std::vector new_optics_groups; - FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDsin[MDs_id]) - { - int tmp; - MDsin[MDs_id].getValue(EMDL_IMAGE_OPTICS_GROUP, tmp); - new_optics_groups.push_back(tmp); - } - - MetaDataTable unique_opticsMdt; - unique_opticsMdt.addMissingLabels(&obsModels[obs_id].opticsMdt); - - FOR_ALL_OBJECTS_IN_METADATA_TABLE(obsModels[obs_id].opticsMdt) - { - std::string myname; - int my_optics_group; - obsModels[obs_id].opticsMdt.getValue(EMDL_IMAGE_OPTICS_GROUP_NAME, myname); - obsModels[obs_id].opticsMdt.getValue(EMDL_IMAGE_OPTICS_GROUP, my_optics_group); - - // Check whether this name is unique - bool is_uniq = true; - int new_group; - for (new_group = 0; new_group < optics_group_uniq_names.size(); new_group++) - { - if (optics_group_uniq_names[new_group] == myname) - { - is_uniq = false; - break; - } - } - new_group ++; // start counting of groups at 1, not 0! - - if (is_uniq) - { - if (verb > 0) std::cout << " + Adding new optics_group with name: " << myname << std::endl; - - optics_group_uniq_names.push_back(myname); - // Add the line to the global obsModel - obsModels[obs_id].opticsMdt.setValue(EMDL_IMAGE_OPTICS_GROUP, new_group); - - unique_opticsMdt.addObject(); - unique_opticsMdt.setObject(obsModels[obs_id].opticsMdt.getObject()); - } - else - { - if (verb > 0) std::cout << " + Joining optics_groups with the same name: " << myname << std::endl; - if (verb > 0) std::cerr << " + WARNING: if these are different data sets, you might want to rename optics groups instead of joining them!" << std::endl; - if (verb > 0) std::cerr << " + WARNING: if so, manually edit the rlnOpticsGroupName column in the optics_groups table of your input STAR files." << std::endl; - } - - if (my_optics_group != new_group) - { - if (verb > 0) std::cout << " + Renumbering group " << myname << " from " << my_optics_group << " to " << new_group << std::endl; - } - - // Update the optics_group entry for all particles in the MDsin - for (long int current_object2 = MDsin[MDs_id].firstObject(); - current_object2 < MDsin[MDs_id].numberOfObjects() && current_object2 >= 0; - current_object2 = MDsin[MDs_id].nextObject()) - { - int old_optics_group; - MDsin[MDs_id].getValue(EMDL_IMAGE_OPTICS_GROUP, old_optics_group, current_object2); - if (old_optics_group == my_optics_group) - new_optics_groups[current_object2] = new_group; - } - } - - obsModels[obs_id].opticsMdt = unique_opticsMdt; - - FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDsin[MDs_id]) - { - MDsin[MDs_id].setValue(EMDL_IMAGE_OPTICS_GROUP, new_optics_groups[current_object]); - - // Also rename the rlnGroupName to not have groups overlapping from different optics groups - std::string name; - if (MDsin[MDs_id].getValue(EMDL_MLMODEL_GROUP_NAME, name)) - { - name = "optics"+integerToString(new_optics_groups[current_object])+"_"+name; - MDsin[MDs_id].setValue(EMDL_MLMODEL_GROUP_NAME, name); - } - } - } - - // Make one vector for combination of the optics tables - MDoptics.push_back(obsModel.opticsMdt); - for (int i = 1; i < fns_in.size(); i++) - { - MDoptics.push_back(obsModels[i - 1].opticsMdt); - } - - // Check if anisotropic magnification and/or beam_tilt are present in some optics groups, but not in others. - // If so, initialise the others correctly - bool has_beamtilt = false, has_not_beamtilt = false; - bool has_anisomag = false, has_not_anisomag = false; - bool has_odd_zernike = false, has_not_odd_zernike = false; - bool has_even_zernike = false, has_not_even_zernike = false; - bool has_ctf_premultiplied = false, has_not_ctf_premultiplied = false; - for (int i = 0; i < fns_in.size(); i++) - { - if (MDoptics[i].containsLabel(EMDL_IMAGE_BEAMTILT_X) || - MDoptics[i].containsLabel(EMDL_IMAGE_BEAMTILT_Y)) - { - has_beamtilt = true; - } - else - { - has_not_beamtilt = true; - } - if (MDoptics[i].containsLabel(EMDL_IMAGE_MAG_MATRIX_00) && - MDoptics[i].containsLabel(EMDL_IMAGE_MAG_MATRIX_01) && - MDoptics[i].containsLabel(EMDL_IMAGE_MAG_MATRIX_10) && - MDoptics[i].containsLabel(EMDL_IMAGE_MAG_MATRIX_11)) - { - has_anisomag = true; - } - else - { - has_not_anisomag = true; - } - if (MDoptics[i].containsLabel(EMDL_IMAGE_ODD_ZERNIKE_COEFFS)) - { - has_odd_zernike = true; - } - else - { - has_not_odd_zernike = true; - } - if (MDoptics[i].containsLabel(EMDL_IMAGE_EVEN_ZERNIKE_COEFFS)) - { - has_even_zernike = true; - } - else - { - has_not_even_zernike = true; - } - if (MDoptics[i].containsLabel(EMDL_OPTIMISER_DATA_ARE_CTF_PREMULTIPLIED)) - { - has_ctf_premultiplied = true; - } - else - { - has_not_ctf_premultiplied = true; - } - } -#ifdef DEBUG - printf("has_beamtilt = %d, has_not_beamtilt = %d, has_anisomag = %d, has_not_anisomag = %d, has_odd_zernike = %d, has_not_odd_zernike = %d, has_even_zernike = %d, has_not_even_zernike = %d, has_ctf_premultiplied = %d, has_not_ctf_premultiplied = %d\n", has_beamtilt, has_not_beamtilt, has_anisomag, has_not_anisomag, has_odd_zernike, has_not_odd_zernike, has_even_zernike, has_not_even_zernike, has_ctf_premultiplied, has_not_ctf_premultiplied); -#endif - - for (int i = 0; i < fns_in.size(); i++) - { - if (has_beamtilt && has_not_beamtilt) - { - if (!MDoptics[i].containsLabel(EMDL_IMAGE_BEAMTILT_X)) - { - FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDoptics[i]) - { - MDoptics[i].setValue(EMDL_IMAGE_BEAMTILT_X, 0.); - } - } - if (!MDoptics[i].containsLabel(EMDL_IMAGE_BEAMTILT_Y)) - { - FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDoptics[i]) - { - MDoptics[i].setValue(EMDL_IMAGE_BEAMTILT_Y, 0.); - } - } - } - - if (has_anisomag && has_not_anisomag) - { - if (!(MDoptics[i].containsLabel(EMDL_IMAGE_MAG_MATRIX_00) && - MDoptics[i].containsLabel(EMDL_IMAGE_MAG_MATRIX_01) && - MDoptics[i].containsLabel(EMDL_IMAGE_MAG_MATRIX_10) && - MDoptics[i].containsLabel(EMDL_IMAGE_MAG_MATRIX_11)) ) - { - FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDoptics[i]) - { - MDoptics[i].setValue(EMDL_IMAGE_MAG_MATRIX_00, 1.); - MDoptics[i].setValue(EMDL_IMAGE_MAG_MATRIX_01, 0.); - MDoptics[i].setValue(EMDL_IMAGE_MAG_MATRIX_10, 0.); - MDoptics[i].setValue(EMDL_IMAGE_MAG_MATRIX_11, 1.); - } - } - } - - if (has_odd_zernike && has_not_odd_zernike) - { - std::vector six_zeros(6, 0); - if (!MDoptics[i].containsLabel(EMDL_IMAGE_ODD_ZERNIKE_COEFFS)) - { - FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDoptics[i]) - { - MDoptics[i].setValue(EMDL_IMAGE_ODD_ZERNIKE_COEFFS, six_zeros); - } - } - } - - if (has_even_zernike && has_not_even_zernike) - { - std::vector nine_zeros(9, 0); - if (!MDoptics[i].containsLabel(EMDL_IMAGE_EVEN_ZERNIKE_COEFFS)) - { - FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDoptics[i]) - { - MDoptics[i].setValue(EMDL_IMAGE_EVEN_ZERNIKE_COEFFS, nine_zeros); - } - } - } - - if (has_ctf_premultiplied && has_not_ctf_premultiplied) - { - if (!MDoptics[i].containsLabel(EMDL_OPTIMISER_DATA_ARE_CTF_PREMULTIPLIED)) - { - FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDoptics[i]) - { - MDoptics[i].setValue(EMDL_OPTIMISER_DATA_ARE_CTF_PREMULTIPLIED, false); - } - } - } - } - - // Now combine all optics tables into one - obsModel.opticsMdt = MetaDataTable::combineMetaDataTables(MDoptics); - } - - // Combine the particles tables - MDout = MetaDataTable::combineMetaDataTables(MDsin); - - //Deactivate the group_name column - MDout.deactivateLabel(EMDL_MLMODEL_GROUP_NO); - - if (fn_check != "") - { - EMDLabel label = EMDL::str2Label(fn_check); - if (!MDout.containsLabel(label)) - REPORT_ERROR("ERROR: the output file does not contain the label to check for duplicates. Is it present in all input files?"); - - /// Don't want to mess up original order, so make a MDsort with only that label... - FileName fn_this, fn_prev = ""; - MetaDataTable MDsort; - FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDout) - { - MDout.getValue(label, fn_this); - MDsort.addObject(); - MDsort.setValue(label, fn_this); - } - // sort on the label - MDsort.newSort(label); - long int nr_duplicates = 0; - FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDsort) - { - MDsort.getValue(label, fn_this); - if (fn_this == fn_prev) - { - nr_duplicates++; - std::cerr << " WARNING: duplicate entry: " << fn_this << std::endl; - } - fn_prev = fn_this; - } - - if (nr_duplicates > 0) - std::cerr << " WARNING: Total number of duplicate "<< fn_check << " entries: " << nr_duplicates << std::endl; - } - - MDout.setName(MDin0.getName()); - -} diff --git a/src/star_handling.h b/src/star_handling.h deleted file mode 100644 index bf94ae1f2..000000000 --- a/src/star_handling.h +++ /dev/null @@ -1,36 +0,0 @@ -/*************************************************************************** - * - * Author: "Sjors H.W. Scheres" - * MRC Laboratory of Molecular Biology - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * This complete copyright notice must be included in any revised version of the - * source code. Additional authorship citations may be added, but existing - * author citations must be preserved. - ***************************************************************************/ - - -#ifndef RELION_STAR_HANDLING_H -#define RELION_STAR_HANDLING_H - -#include -#include -#include - -// Read in all the STAR files in the vector; if do_ignore_optics is true, use tablename_in, otherwise fill obsModel -// fn_check is a string for duplication checking, leave empty for no checking -void combineStarfiles(std::vector &fns_in, - MetaDataTable &MDout, ObservationModel &obsModel, - FileName fn_check = "", bool do_ignore_optics = false, std::string tablename_in = "", int verb = 0); - - -#endif //RELION_STAR_HANDLING_H From fe059c80459046e5942c3bfa526e3567ce0286dc Mon Sep 17 00:00:00 2001 From: scheres Date: Thu, 7 Apr 2022 17:54:13 +0100 Subject: [PATCH 012/495] switch off doseweighting as default in tomo --- src/pipeline_jobs.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/pipeline_jobs.cpp b/src/pipeline_jobs.cpp index 7369a3084..27683f84e 100644 --- a/src/pipeline_jobs.cpp +++ b/src/pipeline_jobs.cpp @@ -1407,8 +1407,12 @@ Note that multiple MotionCor2 processes should not share a GPU; otherwise, it ca joboptions["other_motioncor2_args"] = JobOption("Other MOTIONCOR2 arguments", std::string(""), "Additional arguments that need to be passed to MOTIONCOR2."); // Dose-weight - joboptions["do_dose_weighting"] = JobOption("Do dose-weighting?", true ,"If set to Yes, the averaged micrographs will be dose-weighted."); - joboptions["do_save_noDW"] = JobOption("Save non-dose weighted as well?", false, "Aligned but non-dose weighted images are sometimes useful in CTF estimation, although there is no difference in most cases. Whichever the choice, CTF refinement job is always done on dose-weighted particles."); + if (is_tomo) + joboptions["do_dose_weighting"] = JobOption("Do dose-weighting?", true ,"If set to Yes, the averaged micrographs will be dose-weighted."); + else + joboptions["do_dose_weighting"] = JobOption("Do dose-weighting?", false ,"If set to Yes, the averaged micrographs will be dose-weighted."); + + joboptions["do_save_noDW"] = JobOption("Save non-dose weighted as well?", false, "Aligned but non-dose weighted images are sometimes useful in CTF estimation, although there is no difference in most cases. Whichever the choice, CTF refinement job is always done on dose-weighted particles."); joboptions["dose_per_frame"] = JobOption("Dose per frame (e/A2):", 1, 0, 5, 0.2, "Dose per movie frame (in electrons per squared Angstrom)."); joboptions["pre_exposure"] = JobOption("Pre-exposure (e/A2):", 0, 0, 5, 0.5, "Pre-exposure dose (in electrons per squared Angstrom)."); From e24a71202e24778e77c7f67392bb22889c058150 Mon Sep 17 00:00:00 2001 From: scheres Date: Thu, 7 Apr 2022 17:54:33 +0100 Subject: [PATCH 013/495] remove include of star_handling --- src/jaz/tomography/tomogram_set.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/jaz/tomography/tomogram_set.h b/src/jaz/tomography/tomogram_set.h index 46a44726d..b01dea584 100644 --- a/src/jaz/tomography/tomogram_set.h +++ b/src/jaz/tomography/tomogram_set.h @@ -8,8 +8,6 @@ #include #include #include -#include "src/star_handling.h" - class TomogramSet { From c88b609947ef47935fa47ae2bfaadff1f38cca5c Mon Sep 17 00:00:00 2001 From: scheres Date: Thu, 7 Apr 2022 17:55:30 +0100 Subject: [PATCH 014/495] let reconstruct_tomogram also work on all tomograms in the input, including MPI version and do_only_unfinished --- .../tomography/apps/reconstruct_tomogram.cpp | 1 + .../apps/reconstruct_tomogram_mpi.cpp | 52 +++++++++++ .../programs/reconstruct_tomogram.cpp | 86 ++++++++++++++++--- .../programs/reconstruct_tomogram.h | 15 +++- 4 files changed, 137 insertions(+), 17 deletions(-) create mode 100644 src/jaz/tomography/apps/reconstruct_tomogram_mpi.cpp diff --git a/src/jaz/tomography/apps/reconstruct_tomogram.cpp b/src/jaz/tomography/apps/reconstruct_tomogram.cpp index c1e69e9a7..1c109d433 100644 --- a/src/jaz/tomography/apps/reconstruct_tomogram.cpp +++ b/src/jaz/tomography/apps/reconstruct_tomogram.cpp @@ -9,6 +9,7 @@ int main(int argc, char *argv[]) TomoBackprojectProgram program; program.readParameters(argc, argv); + program.initialise(); program.run(); } catch (RelionError e) diff --git a/src/jaz/tomography/apps/reconstruct_tomogram_mpi.cpp b/src/jaz/tomography/apps/reconstruct_tomogram_mpi.cpp new file mode 100644 index 000000000..036c7c16b --- /dev/null +++ b/src/jaz/tomography/apps/reconstruct_tomogram_mpi.cpp @@ -0,0 +1,52 @@ +/*************************************************************************** + * + * Author: "Sjors H.W. Scheres" + * MRC Laboratory of Molecular Biology + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * This complete copyright notice must be included in any revised version of the + * source code. Additional authorship citations may be added, but existing + * author citations must be preserved. + ***************************************************************************/ +#include +#include +#include + + +int main(int argc, char *argv[]) +{ + try + { + int rank, size; + + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); + // Handle errors + MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN); + + TomoBackprojectProgram program; + + program.readParameters(argc, argv); + program.initialise(); + program.run(rank, size); + } + catch (RelionError e) + { + return RELION_EXIT_FAILURE; + } + + MPI_Barrier(MPI_COMM_WORLD); + MPI_Finalize(); + + return RELION_EXIT_SUCCESS; +} diff --git a/src/jaz/tomography/programs/reconstruct_tomogram.cpp b/src/jaz/tomography/programs/reconstruct_tomogram.cpp index c8f5cec15..7e8148f1e 100644 --- a/src/jaz/tomography/programs/reconstruct_tomogram.cpp +++ b/src/jaz/tomography/programs/reconstruct_tomogram.cpp @@ -3,12 +3,12 @@ #include #include #include -#include #include #include #include #include #include +#include #include @@ -32,13 +32,13 @@ void TomoBackprojectProgram::readParameters(int argc, char *argv[]) int gen_section = parser.addSection("General options"); - tomoName = parser.getOption("--tn", "Tomogram name"); + tomoName = parser.getOption("--tn", "Tomogram name", "*"); applyWeight = !parser.checkOption("--no_weight", "Do not perform weighting in Fourier space using a Wiener filter"); applyPreWeight = parser.checkOption("--pre_weight", "Pre-weight the 2D slices prior to backprojection"); FourierCrop = parser.checkOption("--Fc", "Downsample the 2D images by Fourier cropping"); - - SNR = textToDouble(parser.getOption("--SNR", "SNR assumed by the Wiener filter", "10")); + do_only_unfinished = parser.checkOption("--only_do_unfinished", "Only reconstruct those tomograms that haven't finished yet"); + SNR = textToDouble(parser.getOption("--SNR", "SNR assumed by the Wiener filter", "10")); applyCtf = !parser.checkOption("--noctf", "Ignore the CTF"); @@ -59,7 +59,7 @@ void TomoBackprojectProgram::readParameters(int argc, char *argv[]) n_threads = textToInteger(parser.getOption("--j", "Number of threads", "1")); - outFn = parser.getOption("--o", "Output filename"); + outFn = parser.getOption("--o", "Output filename (or directory in case of reconstructing all tomograms)"); Log::readParams(parser); @@ -75,14 +75,55 @@ void TomoBackprojectProgram::readParameters(int argc, char *argv[]) ZIO::ensureParentDir(outFn); } +void TomoBackprojectProgram::initialise() +{ + if (!tomogramSet.read(optimisationSet.tomograms)) + REPORT_ERROR("ERROR: there was a problem reading the tomogram set"); + + // Make sure output directory exists (this will work for one or all tomograms) + FileName fn_tmp = getOutputFileName(0); + if (!exists(fn_tmp.beforeLastOf("/"))) mktree(fn_tmp.beforeLastOf("/")); + + tomoIndexTodo.clear(); + + if (tomoName == "*") + { + for (int idx = 0; idx < tomogramSet.size(); idx++) + { + if (do_only_unfinished && exists(getOutputFileName(idx))) + continue; + tomoIndexTodo.push_back(idx); + } + } + else + { + tomoIndexTodo.push_back(tomogramSet.getTomogramIndex(tomoName)); + } + + std::cout << " + Reconstructing " << tomoIndexTodo.size() << " tomograms ... " << std::endl; + +} -void TomoBackprojectProgram::run() +void TomoBackprojectProgram::run(int rank, int size) { - TomogramSet tomogramSet(optimisationSet.tomograms); - const int tomoIndex = tomogramSet.getTomogramIndex(tomoName); - Tomogram tomogram = tomogramSet.loadTomogram(tomoIndex, true); - - const int w0 = tomogram.w0; + + long my_first_idx, my_last_idx; + divide_equally(tomoIndexTodo.size(), size, rank , my_first_idx, my_last_idx); + for (long idx = my_first_idx; idx <= my_last_idx; idx++) + { + + reconstructOneTomogram(tomoIndexTodo[idx]); + } + + std::cout << " Done!" << std::endl; + +} + +void TomoBackprojectProgram::reconstructOneTomogram(int tomoIndex) +{ + Tomogram tomogram = tomogramSet.loadTomogram(tomoIndex, true); + + const int w0 = tomogram.w0; const int h0 = tomogram.h0; const int d0 = tomogram.d0; @@ -225,8 +266,25 @@ void TomoBackprojectProgram::run() } Log::print("Writing output"); - - const double samplingRate = tomogram.optics.pixelSize * spacing; - out.write(outFn, samplingRate); + const double samplingRate = tomogramSet.getPixelSize(tomoIndex) * spacing; + out.write(getOutputFileName(), samplingRate); + } + +FileName TomoBackprojectProgram::getOutputFileName(int index) +{ + if (tomoName == "*") + { + FileName result = outFn; + if (result[result.size()-1] != '/') result += "/"; + result += "tomograms/rec_" + tomogramSet.getTomogramName(index)+".mrc"; + return result; + } + else + { + return outFn; + } + + +} \ No newline at end of file diff --git a/src/jaz/tomography/programs/reconstruct_tomogram.h b/src/jaz/tomography/programs/reconstruct_tomogram.h index 6c31c6548..58f0a1f03 100644 --- a/src/jaz/tomography/programs/reconstruct_tomogram.h +++ b/src/jaz/tomography/programs/reconstruct_tomogram.h @@ -3,9 +3,11 @@ #include #include +#include #include #include #include +#include class TomoBackprojectProgram { @@ -18,13 +20,20 @@ class TomoBackprojectProgram double spacing, x0, y0, z0, taperDist, taperFalloff; std::string tomoName, outFn; bool applyPreWeight, applyWeight, applyCtf, zeroDC, FourierCrop; + bool do_only_unfinished; double SNR; + std::vector tomoIndexTodo; OptimisationSet optimisationSet; - - + TomogramSet tomogramSet; + void readParameters(int argc, char *argv[]); - void run(); + void initialise(); + void run(int rank = 0, int size = 1); + void reconstructOneTomogram(int tomoIndex); + + private: + FileName getOutputFileName(int index = -1); }; #endif From 3b7d9e661b2b51643f6cabb080b00f0566dfff2d Mon Sep 17 00:00:00 2001 From: scheres Date: Fri, 8 Apr 2022 16:43:05 +0100 Subject: [PATCH 015/495] add abort options and progress bar --- .../programs/reconstruct_tomogram.cpp | 29 +++++++++++++++++-- .../programs/reconstruct_tomogram.h | 3 +- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/jaz/tomography/programs/reconstruct_tomogram.cpp b/src/jaz/tomography/programs/reconstruct_tomogram.cpp index 7e8148f1e..74ba01cd2 100644 --- a/src/jaz/tomography/programs/reconstruct_tomogram.cpp +++ b/src/jaz/tomography/programs/reconstruct_tomogram.cpp @@ -106,13 +106,34 @@ void TomoBackprojectProgram::initialise() void TomoBackprojectProgram::run(int rank, int size) { - long my_first_idx, my_last_idx; divide_equally(tomoIndexTodo.size(), size, rank , my_first_idx, my_last_idx); + + + int barstep, nr_todo = my_last_idx-my_first_idx+1; + if (rank == 0) + { + init_progress_bar(nr_todo); + barstep = XMIPP_MAX(1, nr_todo / 60); + } for (long idx = my_first_idx; idx <= my_last_idx; idx++) { + // Abort through the pipeline_control system + if (pipeline_control_check_abort_job()) + exit(RELION_EXIT_ABORTED); reconstructOneTomogram(tomoIndexTodo[idx]); + + if (idx % barstep == 0) + progress_bar(idx); + } + + progress_bar(nr_todo); + + // If the output filename was a directory, then also write updated tomograms.star. + if (outFn[outFn.size()-1] == '/') + { + tomogramSet.write(outFn + "tomograms.star"); } std::cout << " Done!" << std::endl; @@ -270,11 +291,15 @@ void TomoBackprojectProgram::reconstructOneTomogram(int tomoIndex) const double samplingRate = tomogramSet.getPixelSize(tomoIndex) * spacing; out.write(getOutputFileName(), samplingRate); + // Also add the tomogram name to the tomogramSet + tomogramSet.globalTable.setValue(EMDL_TOMO_RECONSTRUCTED_TOMOGRAM_FILE_NAME, getOutputFileName(), tomoIndex); + } FileName TomoBackprojectProgram::getOutputFileName(int index) { - if (tomoName == "*") + // If we're reconstructing many tomograms, or the output filename is a directory: use standardized output filenames + if (tomoName == "*" || outFn[outFn.size()-1] == '/') { FileName result = outFn; if (result[result.size()-1] != '/') result += "/"; diff --git a/src/jaz/tomography/programs/reconstruct_tomogram.h b/src/jaz/tomography/programs/reconstruct_tomogram.h index 58f0a1f03..2cea6a0dd 100644 --- a/src/jaz/tomography/programs/reconstruct_tomogram.h +++ b/src/jaz/tomography/programs/reconstruct_tomogram.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -18,7 +19,7 @@ class TomoBackprojectProgram int n_threads; int w, h, d; double spacing, x0, y0, z0, taperDist, taperFalloff; - std::string tomoName, outFn; + FileName tomoName, outFn; bool applyPreWeight, applyWeight, applyCtf, zeroDC, FourierCrop; bool do_only_unfinished; double SNR; From 2c0a24820c7c7e4b615096194b4a160b5e0e769a Mon Sep 17 00:00:00 2001 From: scheres Date: Fri, 8 Apr 2022 16:44:03 +0100 Subject: [PATCH 016/495] git add support for reconstruction_tomograns, import tiltseries (Alister) and tiltseries alignment to the GUI --- src/gui_jobwindow.cpp | 134 ++++++++++++++++++++++++---- src/gui_jobwindow.h | 6 +- src/gui_mainwindow.cpp | 20 ++++- src/metadata_label.h | 5 +- src/pipeline_jobs.cpp | 195 ++++++++++++++++++++++++++++++++++++++--- src/pipeline_jobs.h | 36 ++++++-- 6 files changed, 359 insertions(+), 37 deletions(-) diff --git a/src/gui_jobwindow.cpp b/src/gui_jobwindow.cpp index fbb4e2a32..bf2e8e6cb 100644 --- a/src/gui_jobwindow.cpp +++ b/src/gui_jobwindow.cpp @@ -414,6 +414,16 @@ void JobWindow::initialise(int my_job_type, bool _is_tomo) myjob.initialise(my_job_type); initialiseTomoImportWindow(); } + else if (my_job_type == PROC_TOMO_RECONSTRUCT_TOMOGRAM) + { + myjob.initialise(my_job_type); + initialiseTomoReconstructTomogramsWindow(); + } + else if (my_job_type == PROC_TOMO_ALIGN_TILTSERIES) + { + myjob.initialise(my_job_type); + initialiseTomoAlignTiltseriesWindow(); + } else if (my_job_type == PROC_TOMO_SUBTOMO) { myjob.initialise(my_job_type); @@ -2353,10 +2363,42 @@ void JobWindow::placeTomoInput(bool has_tomograms, bool has_particles, void JobWindow::initialiseTomoImportWindow() { - setupTabs(3); + setupTabs(5); - tab1->begin(); - tab1->label("Tomograms"); + tab1->begin(); + tab1->label("General"); + resetHeight(); + + place("angpix", TOGGLE_DEACTIVATE); + place("kV", TOGGLE_DEACTIVATE); + place("Cs", TOGGLE_DEACTIVATE); + place("Q0", TOGGLE_DEACTIVATE); + place("dose", TOGGLE_DEACTIVATE); + + tab1->end(); + + tab2->begin(); + tab2->label("Tilt series"); + resetHeight(); + + group4 = new Fl_Group(WCOL0, MENUHEIGHT, 550, 600-MENUHEIGHT, ""); + group4->end(); + place("do_tiltseries", TOGGLE_DEACTIVATE, group4, false); + group4->begin(); + + place("movie_files", TOGGLE_DEACTIVATE); + place("mdoc_files", TOGGLE_DEACTIVATE); + place("prefix", TOGGLE_DEACTIVATE); + + current_y += STEPY/2; + place("tilt_axis_angle", TOGGLE_DEACTIVATE); + place("mtf_file", TOGGLE_DEACTIVATE); + + group4->end(); + guientries["do_tiltseries"].cb_menu_i(); // make default active + + tab3->begin(); + tab3->label("Tomograms"); resetHeight(); group1 = new Fl_Group(WCOL0, MENUHEIGHT, 550, 600-MENUHEIGHT, ""); @@ -2366,15 +2408,10 @@ void JobWindow::initialiseTomoImportWindow() place("tomo_star", TOGGLE_DEACTIVATE); place("io_tomos", TOGGLE_DEACTIVATE); - place("angpix", TOGGLE_DEACTIVATE); - place("kV", TOGGLE_DEACTIVATE); - place("Cs", TOGGLE_DEACTIVATE); - place("Q0", TOGGLE_DEACTIVATE); // Add a little spacer current_y += STEPY/2; - place("dose", TOGGLE_DEACTIVATE); place("order_list", TOGGLE_DEACTIVATE); place("do_flipYZ", TOGGLE_DEACTIVATE); place("do_flipZ", TOGGLE_DEACTIVATE); @@ -2383,10 +2420,10 @@ void JobWindow::initialiseTomoImportWindow() group1->end(); guientries["do_tomo"].cb_menu_i(); // make default active - tab1->end(); + tab3->end(); - tab2->begin(); - tab2->label("Coordinates"); + tab4->begin(); + tab4->label("Coordinates"); resetHeight(); group2 = new Fl_Group(WCOL0, MENUHEIGHT, 550, 600-MENUHEIGHT, ""); @@ -2403,10 +2440,10 @@ void JobWindow::initialiseTomoImportWindow() group2->end(); guientries["do_coords"].cb_menu_i(); // make default active - tab2->end(); + tab4->end(); - tab3->begin(); - tab3->label("Others"); + tab5->begin(); + tab5->label("Others"); resetHeight(); group3 = new Fl_Group(WCOL0, MENUHEIGHT, 550, 600-MENUHEIGHT, ""); @@ -2429,7 +2466,74 @@ void JobWindow::initialiseTomoImportWindow() group3->end(); guientries["do_other"].cb_menu_i(); // make default active - tab3->end(); + tab5->end(); +} + +void JobWindow::initialiseTomoAlignTiltseriesWindow() +{ + setupTabs(1); + + tab1->begin(); + tab1->label("I/O"); + resetHeight(); + + place("in_tiltseries", TOGGLE_DEACTIVATE); + + // Add a little spacer + current_y += STEPY/2; + + group1 = new Fl_Group(WCOL0, MENUHEIGHT, 550, 600-MENUHEIGHT, ""); + group1->end(); + place("do_imod_fiducials", TOGGLE_DEACTIVATE, group1, false); + group1->begin(); + + place("fiducial_diameter", TOGGLE_DEACTIVATE); + group1->end(); + guientries["do_imod_fiducials"].cb_menu_i(); // make default active + + // Add a little spacer + current_y += STEPY/2; + + group2 = new Fl_Group(WCOL0, MENUHEIGHT, 550, 600-MENUHEIGHT, ""); + group2->end(); + place("do_imod_patchtrack", TOGGLE_DEACTIVATE, group2, false); + group2->begin(); + + place("patch_size", TOGGLE_DEACTIVATE); + place("patch_overlap", TOGGLE_DEACTIVATE); + group2->end(); + guientries["do_imod_patchtrack"].cb_menu_i(); // make default active + + tab1->end(); + + +} + +void JobWindow::initialiseTomoReconstructTomogramsWindow() +{ + setupTabs(2); + + placeTomoInput(true, false, false, false, false, false); + + tab2->begin(); + tab2->label("Reconstruct"); + resetHeight(); + + + place("binning", TOGGLE_DEACTIVATE); + + current_y += STEPY /2 ; + + place("xdim", TOGGLE_DEACTIVATE); + place("ydim", TOGGLE_DEACTIVATE); + place("zdim", TOGGLE_DEACTIVATE); + + current_y += STEPY /2 ; + + place("tomo_name"); + + tab2->end(); + } void JobWindow::initialiseTomoSubtomoWindow() diff --git a/src/gui_jobwindow.h b/src/gui_jobwindow.h index 9da752da0..83938c6c0 100644 --- a/src/gui_jobwindow.h +++ b/src/gui_jobwindow.h @@ -129,11 +129,13 @@ class JobWindow : public Fl_Box void initialiseCtfrefineWindow(); void initialiseExternalWindow(); - // relion-3.2: add subtomogram averaging programs by Jasenko + // relion-4.0: add subtomogram averaging programs by Jasenko void placeTomoInput(bool has_tomograms, bool has_particles, bool has_trajectories, bool has_manifolds, bool has_halfmaps, bool has_postprocess); void initialiseTomoImportWindow(); - void initialiseTomoSubtomoWindow(); + void initialiseTomoReconstructTomogramsWindow(); + void initialiseTomoAlignTiltseriesWindow(); + void initialiseTomoSubtomoWindow(); void initialiseTomoCtfRefineWindow(); void initialiseTomoAlignWindow(); void initialiseTomoReconParWindow(); diff --git a/src/gui_mainwindow.cpp b/src/gui_mainwindow.cpp index b2fe45638..32b168aa7 100644 --- a/src/gui_mainwindow.cpp +++ b/src/gui_mainwindow.cpp @@ -408,14 +408,12 @@ GuiMainWindow::GuiMainWindow(int w, int h, const char* title, FileName fn_pipe, if (_do_tomo) { - /* SHWS 6Apr2022: with new motioncorr and ctffind jobtypes, the old Import makes no sesne anymore, so removed for now - browse_grp[nr_browse_tabs] = new Fl_Group(WCOL0, 2, 550, 615-MENUHEIGHT); + browse_grp[nr_browse_tabs] = new Fl_Group(WCOL0, 2, 550, 615-MENUHEIGHT); browser->add("Tomo import"); gui_jobwindows[nr_browse_tabs] = new JobWindow(); gui_jobwindows[nr_browse_tabs]->initialise(PROC_TOMO_IMPORT); browse_grp[nr_browse_tabs]->end(); nr_browse_tabs++; - */ browse_grp[nr_browse_tabs] = new Fl_Group(WCOL0, 2, 550, 615-MENUHEIGHT); browser->add("Motion correction"); @@ -431,7 +429,21 @@ GuiMainWindow::GuiMainWindow(int w, int h, const char* title, FileName fn_pipe, browse_grp[nr_browse_tabs]->end(); nr_browse_tabs++; - browse_grp[nr_browse_tabs] = new Fl_Group(WCOL0, 2, 550, 615-MENUHEIGHT); + browse_grp[nr_browse_tabs] = new Fl_Group(WCOL0, 2, 550, 615-MENUHEIGHT); + browser->add("Align tiltseries"); + gui_jobwindows[nr_browse_tabs] = new JobWindow(); + gui_jobwindows[nr_browse_tabs]->initialise(PROC_TOMO_ALIGN_TILTSERIES); + browse_grp[nr_browse_tabs]->end(); + nr_browse_tabs++; + + browse_grp[nr_browse_tabs] = new Fl_Group(WCOL0, 2, 550, 615-MENUHEIGHT); + browser->add("Reconstruct tomograms"); + gui_jobwindows[nr_browse_tabs] = new JobWindow(); + gui_jobwindows[nr_browse_tabs]->initialise(PROC_TOMO_RECONSTRUCT_TOMOGRAM); + browse_grp[nr_browse_tabs]->end(); + nr_browse_tabs++; + + browse_grp[nr_browse_tabs] = new Fl_Group(WCOL0, 2, 550, 615-MENUHEIGHT); browser->add("Make pseudo-subtomos"); gui_jobwindows[nr_browse_tabs] = new JobWindow(); gui_jobwindows[nr_browse_tabs]->initialise(PROC_TOMO_SUBTOMO); diff --git a/src/metadata_label.h b/src/metadata_label.h index e235687a5..5c5a77a22 100644 --- a/src/metadata_label.h +++ b/src/metadata_label.h @@ -606,7 +606,8 @@ enum EMDLabel EMDL_TOMO_TILT_SERIES_NAME, EMDL_TOMO_TILT_SERIES_STARFILE, EMDL_TOMO_FRAME_COUNT, - EMDL_TOMO_SIZE_X, + EMDL_TOMO_RECONSTRUCTED_TOMOGRAM_FILE_NAME, + EMDL_TOMO_SIZE_X, EMDL_TOMO_SIZE_Y, EMDL_TOMO_SIZE_Z, EMDL_TOMO_PROJECTION_X, @@ -1289,6 +1290,8 @@ class StaticInitialization EMDL::addLabel(EMDL_TOMO_TILT_SERIES_NAME, EMDL_STRING, "rlnTomoTiltSeriesName", "Tilt series file name"); EMDL::addLabel(EMDL_TOMO_TILT_SERIES_STARFILE, EMDL_STRING, "rlnTomoTiltSeriesStarFile", "Tilt series starfile"); EMDL::addLabel(EMDL_TOMO_FRAME_COUNT, EMDL_INT, "rlnTomoFrameCount", "Number of tilts in a tilt series"); + EMDL::addLabel(EMDL_TOMO_RECONSTRUCTED_TOMOGRAM_FILE_NAME, EMDL_STRING, "rlnTomoReconstructedTomogram", "File name of a reconstructed tomogram"); + EMDL::addLabel(EMDL_TOMO_FRAME_COUNT, EMDL_INT, "rlnTomoFrameCount", "Number of tilts in a tilt series"); EMDL::addLabel(EMDL_TOMO_SIZE_X, EMDL_INT, "rlnTomoSizeX", "Width of a bin-1 tomogram in pixels"); EMDL::addLabel(EMDL_TOMO_SIZE_Y, EMDL_INT, "rlnTomoSizeY", "Height of a bin-1 tomogram in pixels"); EMDL::addLabel(EMDL_TOMO_SIZE_Z, EMDL_INT, "rlnTomoSizeZ", "Depth of a bin-1 tomogram in pixels"); diff --git a/src/pipeline_jobs.cpp b/src/pipeline_jobs.cpp index 27683f84e..ce7568e7b 100644 --- a/src/pipeline_jobs.cpp +++ b/src/pipeline_jobs.cpp @@ -481,6 +481,8 @@ bool RelionJob::read(std::string fn, bool &_is_continue, bool do_initialise) type != PROC_TOMO_SUBTOMO && type != PROC_TOMO_CTFREFINE && type != PROC_TOMO_ALIGN && + type != PROC_TOMO_ALIGN_TILTSERIES && + type != PROC_TOMO_RECONSTRUCT_TOMOGRAM && type != PROC_TOMO_RECONSTRUCT && type != PROC_EXTERNAL) REPORT_ERROR("ERROR: cannot find correct job type in " + myfilename + "run.job, with type= " + integerToString(type)); @@ -876,6 +878,16 @@ void RelionJob::initialise(int _job_type) has_mpi = has_thread = false; initialiseTomoImportJob(); } + else if (type == PROC_TOMO_ALIGN_TILTSERIES) + { + has_mpi = has_thread = true; + initialiseTomoAlignTiltSeriesJob(); + } + else if (type == PROC_TOMO_RECONSTRUCT_TOMOGRAM) + { + has_mpi = has_thread = true; + initialiseTomoReconstructTomogramsJob(); + } else if (type == PROC_TOMO_SUBTOMO) { has_mpi = has_thread = true; @@ -1127,6 +1139,10 @@ bool RelionJob::getCommands(std::string &outputname, std::vector &c { result = getCommandsTomoAlignJob(outputname, commands, final_command, do_makedir, job_counter, error_message); } + else if (type == PROC_TOMO_ALIGN_TILTSERIES) + { + result = getCommandsTomoAlignTiltSeriesJob(outputname, commands, final_command, do_makedir, job_counter, error_message); + } else if (type == PROC_TOMO_RECONSTRUCT) { result = getCommandsTomoReconPartJob(outputname, commands, final_command, do_makedir, job_counter, @@ -6077,6 +6093,19 @@ void RelionJob::initialiseTomoImportJob() { hidden_name = ".gui_tomo_import"; + joboptions["angpix"] = JobOption("Pixel size (Angstrom):", 1.35, 0.5, 10.0, 0.1, "Pixel size in Angstroms. "); + joboptions["kV"] = JobOption("Voltage (kV):", 300, 80, 300, 10, "Voltage the microscope was operated on (in kV; default=300)."); + joboptions["Cs"] = JobOption("Spherical aberration (mm):", 2.7, 0.01, 4, 0.1 , "Spherical aberration of the microscope used to collect these images (in mm). Typical values are 2.7 (FEI Titan & Talos, most JEOL CRYO-ARM), 2.0 (FEI Polara), 1.4 (some JEOL CRYO-ARM) and 0.01 (microscopes with a Cs corrector)."); + joboptions["Q0"] = JobOption("Amplitude contrast:", 0.1, 0.05, 1, 0.01, "Fraction of amplitude contrast (default=0.1). Often values around 10% work better than theoretically more accurate lower values. "); + joboptions["dose"] = JobOption("Frame dose:", 3.0, 0.0, 10.0, 0.1 , "Electron dose (in e/A^2) per frame (image) in the tilt series."); + + joboptions["do_tiltseries"]= JobOption("Import tiltseries?", true, "Set this to Yes for importing tilt series straight from serialEM."); + joboptions["movie_files"] = JobOption("Tilt image movie files:", (std::string)"frames/*mrc","File pattern pointing to the raw movie files for the tilt images"); + joboptions["mdoc_files"] = JobOption("mdoc files:", (std::string)"mdoc/*.mdoc","File pattern pointing to the mdoc files from the data acquisition software"); + joboptions["prefix"] = JobOption("Prefix:", (std::string)"","Prefix for XXX"); + joboptions["tilt_axis_angle"] = JobOption("Tilt axis angle (deg):", 90.0, 0.0, 180.0, 1.0 , "Nominal value for the angle of the tilt axis"); + joboptions["mtf_file"] = JobOption("MTF file:", (std::string)"","MTF file for the detector"); + joboptions["do_tomo"] = JobOption("Import tomograms?", true, "Set this to Yes for importing tomogram directories from IMOD."); joboptions["io_tomos"] = JobOption("Append to tomograms set: ", OUTNODE_TOMO_TOMOGRAMS, "", "Tomogram set STAR file (*.star)", "The imported tomograms will be output into this tomogram set. If any tomograms were already in this tomogram set, then the newly imported ones will be added to those."); joboptions["tomo_star"] = JobOption("STAR file with tomograms description: ", "", "Input file (*.star)", ".", "Provide a STAR file with the basic following information to import tomogsrams: \n\n" @@ -6090,20 +6119,17 @@ void RelionJob::initialiseTomoImportJob() " - rlnOpticsGroupName: an arbitrary name for an optics group. This allows the set of tilt series to be separated into subsets that share the same optical aberrations. This is useful if the data have been collected in multiple sessions that might exhibit different aberrations. If omitted, all tilt series will be assigned to the same default optics group.\n" " - rlnTomoImportOffset: an arbitrary offset to the 3D coordinate system. This is useful if particles have already been picked in tomograms that have been cropped after reconstruction by IMOD. If the IMOD-internal SHIFT command has been used to apply offsets, then this will be handled internally and does not need to be specified here. If omitted, then the values of the --off command line arguments will be used instead (which default to 0).\n" " - rlnTomoImportCulledFile: output file name for a new tilt series with the excluded frames missing. This is only needed if tilt images have been excluded using IMOD’s EXCLUDE, EXCLUDELIST or EXCLUDELIST2 commands. In that case, this becomes a mandatory parameter."); - joboptions["angpix"] = JobOption("Pixel size (Angstrom):", (std::string)"", "Pixel size in Angstroms. If this values varies among the input tomograms, then specify it using its own column (rlnTomoTiltSeriesPixelSize) in the input tomogram description STAR file."); - joboptions["kV"] = JobOption("Voltage (kV):", (std::string)"", "Voltage the microscope was operated on (in kV; default=300). If this values varies among the input tomograms, then specify it using its own column (rlnVoltage) in the input tomogram description STAR file."); - joboptions["Cs"] = JobOption("Spherical aberration (mm):", (std::string)"", "Spherical aberration of the microscope used to collect these images (in mm; default=2.7). Typical values are 2.7 (FEI Titan & Talos, most JEOL CRYO-ARM), 2.0 (FEI Polara), 1.4 (some JEOL CRYO-ARM) and 0.01 (microscopes with a Cs corrector). If this values varies among the input tomograms, then specify it using its own column (rlnSphericalAberration) in the input tomogram description STAR file."); - joboptions["Q0"] = JobOption("Amplitude contrast:", (std::string)"", "Fraction of amplitude contrast (default=0.1). Often values around 10% work better than theoretically more accurate lower values. If this values varies among the input tomograms, then specify it using its own column (rlnAmplitudeContrast) in the input tomogram description STAR file."); - joboptions["dose"] = JobOption("Frame dose:", (std::string)"", "Electron dose (in e/A^2) per frame (image) in the tilt series. If this values varies among the input tomograms, then specify it using its own column (rlnTomoImportFractionalDose) in the input tomogram description STAR file."); joboptions["order_list"] = JobOption("Ordered list:", (std::string)"", "", ".", "A 2-column, comma-separated file with the frame-order list of the tilt series, where the first column is the frame (image) number (starting at 1) and the second column is the tilt angle (in degrees). If this values varies among the input tomograms, then specify it using its own column (rlnTomoImportOrderList) in the input tomogram description STAR file."); joboptions["do_flipYZ"] = JobOption("Flip YZ?", true, "Set this to Yes if you want to interchange the Y and Z coordinates. If this values varies among the input tomograms, then append opposite values to tomogram set using another Import tomo job."); joboptions["do_flipZ"] = JobOption("Flip Z?", true, "Set this to Yes if you want to change the sign of the Z coordinates. If this values varies among the input tomograms, then append opposite values to tomogram set using another Import tomo job."); joboptions["hand"] = JobOption("Tilt handedness:", (std::string)"", "Set this to indicate the handedness of the tilt geometry (default=-1). The value of this parameter is either +1 or -1, and it describes whether the focus increases or decreases as a function of Z distance. It has to be determined experimentally. In our experiments, it has always been -1. Y If this values varies among the input tomograms, then append opposite values to tomogram set using another Import tomo job."); - joboptions["do_coords"] = JobOption("Import coordinates?", false, "Set this to Yes for importing particle coordinates."); + + joboptions["do_coords"] = JobOption("Import coordinates?", false, "Set this to Yes for importing particle coordinates."); joboptions["part_star"] = JobOption("STAR file with coordinates: ", "", "Input file (*.star)", ".", "Provide a STAR file with the following information to input particles: \n \n TODO TODO TODO "); joboptions["part_tomos"] = JobOption("Tomograms set: ", OUTNODE_TOMO_TOMOGRAMS, "", "Tomogram set STAR file (*.star)", "The tomograms set from which these particles were picked."); joboptions["do_coords_flipZ"] = JobOption("Flip Z coordinates?", false, "Set this to Yes if you want to flip particles Z coordinate. Use it in case imported tomograms Z axis are flipped compared to tomograms used for picking."); - joboptions["do_other"] = JobOption("Import other node types?", false, "Set this to Yes if you plan to import anything else than movies or micrographs"); + + joboptions["do_other"] = JobOption("Import other node types?", false, "Set this to Yes if you plan to import anything else than movies or micrographs"); joboptions["fn_in_other"] = JobOption("Input file:", "ref.mrc", "Input file (*.*)", ".", "Select any file(s) to import. \n \n \ Note that for importing coordinate files, one has to give a Linux wildcard, where the *-symbol is before the coordinate-file suffix, e.g. if the micrographs are called mic1.mrc and the coordinate files mic1.box or mic1_autopick.star, one HAS to give '*.box' or '*_autopick.star', respectively.\n \n \ Also note that micrographs, movies and coordinate files all need to be in the same directory (with the same rootnames, e.g.mic1 in the example above) in order to be imported correctly. 3D masks or references can be imported from anywhere. \n \n \ @@ -6127,22 +6153,45 @@ bool RelionJob::getCommandsTomoImportJob(std::string &outputname, std::vector &commands, + std::string &final_command, bool do_makedir, int job_counter, std::string &error_message) +{ + commands.clear(); + initialisePipeline(outputname, job_counter); + std::string command; + + int i = 0; + if (joboptions["do_imod_fiducials"].getBoolean()) i++; + if (joboptions["do_imod_patchtrack"].getBoolean()) i++; + + if (i != 1) + { + error_message = "ERROR: you should (only) select ONE of the alignment methods: IMOD:fiducials or IMOD:patchtracking."; + return false; + } + + // TODO: rename command to relion_tomo_import_tiltseries? + command = "relion_tomo_align_tilt_series "; + // Make sure the methods are the first argument to the program! + if (joboptions["do_imod_fiducials"].getBoolean()) + { + command += " IMOD:fiducials"; + command += " --nominal-fiducial-diameter-nanometres " + joboptions["fiducial_diameter"].getString(); + } + else if (joboptions["do_imod_patchtrack"].getBoolean()) + { + command += " IMOD:patch-tracking"; + command += " --unbinned-patch-size-pixels " + joboptions["patch_size"].getString(); + command += " -- patch-overlap-percentage " + joboptions["patch_overlap"].getString(); + } + + command += " --tilt-series-star-file " + joboptions["in_tiltseries"].getString(); + Node node(joboptions["in_tiltseries"].getString(), joboptions["in_tiltseries"].node_type); + inputNodes.push_back(node); + + command += " --output-directory " + outputname; + Node node2(outputname+"tilt_series.star", LABEL_TOMO_TOMOGRAMS); + outputNodes.push_back(node2); + + // Other arguments for extraction + command += " " + joboptions["other_args"].getString(); + commands.push_back(command); + + return prepareFinalCommand(outputname, commands, final_command, do_makedir, error_message); + +} + +void RelionJob::initialiseTomoReconstructTomogramsJob() +{ + hidden_name = ".gui_tomo_reconstruct_tomograms"; + + addTomoInputOptions(true, false, false, false, false, false); + + joboptions["tomo_name"] = JobOption("Reconstruct only this tomogram:", std::string(""), "If not left empty, the program will only reconstruct this particular tomogram"); + + joboptions["binning"] = JobOption("Binning factor: ", 8, 1, 32, 1, "The tomogram will be downscaled to this factor. For particle picking, often binning to 4 or 8 gives good enough tomograms. Later subtomogram averaging will not use these binned tomograms."); + + joboptions["xdim"] = JobOption("Tomogram width (Xdim): ", -1, -1, 6000, 100, "The tomogram X-dimension in pixels. If a negative value is given, this will be determined from the size of the tilt series images."); + joboptions["ydim"] = JobOption("Tomogram height (Ydim): ", -1, -1, 6000, 100, "The tomogram Y-dimension in pixels. If a negative value is given, this will be determined from the size of the tilt series images."); + joboptions["zdim"] = JobOption("Tomogram thickness (Zdim): ", 2000, -1, 6000, 100, "The tomogram Z-dimension in pixels. If a negative value is given, this will be determined from the size of the tilt series images."); + + +} +bool RelionJob::getCommandsTomoReconstructTomogramsJob(std::string &outputname, std::vector &commands, + std::string &final_command, bool do_makedir, int job_counter, std::string &error_message) +{ + commands.clear(); + initialisePipeline(outputname, job_counter); + std::string command; + + if (joboptions["nr_mpi"].getNumber(error_message) > 1) + command="`which relion_tomo_reconstruct_tomogram_mpi`"; + else + command="`which relion_tomo_reconstruct_tomogram`"; + if (error_message != "") return false; + + // I/O + error_message = getTomoInputCommmand(command, HAS_COMPULSORY, HAS_NOT, HAS_NOT, HAS_NOT, HAS_NOT, + HAS_NOT); + if (error_message != "") return false; + + command += " --o " + outputname; + command += " --tn " + joboptions["tomo_name"].getString(); + + command += " --w " + joboptions["xdim"].getString(); + command += " --h " + joboptions["ydim"].getString(); + command += " --d " + joboptions["zdim"].getString(); + + Node node1(outputname+"tomograms.star", LABEL_TOMO_TOMOGRAMS); + outputNodes.push_back(node1); + + if (is_continue) + { + command += " --only_do_unfinished "; + } + + // Running stuff + command += " --j " + joboptions["nr_threads"].getString(); + + // Other arguments for extraction + command += " " + joboptions["other_args"].getString(); + commands.push_back(command); + + return prepareFinalCommand(outputname, commands, final_command, do_makedir, error_message); + +} + + + void RelionJob::initialiseTomoSubtomoJob() { diff --git a/src/pipeline_jobs.h b/src/pipeline_jobs.h index 6b6b722a1..8d9d0f3ac 100644 --- a/src/pipeline_jobs.h +++ b/src/pipeline_jobs.h @@ -142,6 +142,8 @@ static const std::vector job_tomo_align_def_model{ "Fourier" }; + + // To have a line on the GUI to change the minimum number of dedicated in a job static bool do_allow_change_minimum_dedicated; @@ -176,7 +178,7 @@ static bool do_allow_change_minimum_dedicated; #define NODE_POLISH_PARAMS 15// Txt file with optimal parameters for Bayesian polishing #define NODE_TOMO_OPTIMISATION 50 // Jasenko's combined optimisation set for subtomogram averaging -#define NODE_TOMO_TOMOGRAMS 51 // Jasenko's set of tomograms +//#define NODE_TOMO_TOMOGRAMS 51 // Jasenko's set of tomograms #define NODE_TOMO_TRAJECTORIES 52 // Jasenko's definition of subtomogram motion trajectories #define NODE_TOMO_MANIFOLDS 53 // Jasenko's definition of 3D shapes (for picking of subtomograms) @@ -672,6 +674,8 @@ static int get_node_type(std::string label) #define PROC_TOMO_ALIGN_DIRNAME "FrameAlignTomo" // Frame alignment and particle polishing for subtomography #define PROC_TOMO_RECONSTRUCT_DIRNAME "ReconstructParticleTomo" // Calculation of particle average from the individual tilt series images #define PROC_EXTERNAL_DIRNAME "External" // For running non-relion programs +#define PROC_TOMO_ALIGN_TILTSERIES_DIRNAME "TiltSeriesAlign" // Tilt series alignment for tomogram reconstruction +#define PROC_TOMO_RECONSTRUCT_TOMOGRAM_DIRNAME "ReconstructTomograms" // Reconstruction of tomograms for particle picking // All the directory names of the different types of jobs defined inside the pipeline #define PROC_IMPORT_LABELNEW "relion.import" // Import any file as a Node of a given type @@ -699,6 +703,8 @@ static int get_node_type(std::string label) #define PROC_TOMO_ALIGN_LABELNEW "relion.framealigntomo" // Frame alignment and particle polishing for subtomography #define PROC_TOMO_RECONSTRUCT_LABELNEW "relion.reconstructparticletomo" // Calculation of particle average from the individual tilt series images #define PROC_EXTERNAL_LABELNEW "relion.external" // For running non-relion programs +#define PROC_TOMO_ALIGN_TILTSERIES_LABELNEW "relion.aligntiltseries" // Tilt series alignment for tomogram reconstruction +#define PROC_TOMO_RECONSTRUCT_TOMOGRAM_LABELNEW "relion.reconstructtomograms" // Reconstruction of tomograms for particle picking #define PROC_IMPORT 0 // Import any file as a Node of a given type @@ -728,6 +734,9 @@ static int get_node_type(std::string label) #define PROC_TOMO_CTFREFINE 52// CTF refinement (defocus & aberrations for tomography) #define PROC_TOMO_ALIGN 53// Frame alignment and particle polishing for subtomography #define PROC_TOMO_RECONSTRUCT 54// Calculation of particle average from the individual tilt series images +#define PROC_TOMO_ALIGN_TILTSERIES 55// Tilt series alignment for tomogram reconstruction +#define PROC_TOMO_RECONSTRUCT_TOMOGRAM 56 // Reconstruction of tomograms for particle picking + #define PROC_EXTERNAL 99// External scripts @@ -755,6 +764,8 @@ static std::map proc_type2dirname = {{PROC_IMPORT, PROC_IMPORT {PROC_TOMO_CTFREFINE, PROC_TOMO_CTFREFINE_DIRNAME}, {PROC_TOMO_ALIGN, PROC_TOMO_ALIGN_DIRNAME}, {PROC_TOMO_RECONSTRUCT, PROC_TOMO_RECONSTRUCT_DIRNAME}, + {PROC_TOMO_ALIGN_TILTSERIES, PROC_TOMO_ALIGN_TILTSERIES_DIRNAME}, + {PROC_TOMO_RECONSTRUCT_TOMOGRAM, PROC_TOMO_RECONSTRUCT_TOMOGRAM_DIRNAME}, {PROC_EXTERNAL, PROC_EXTERNAL_DIRNAME}}; static std::map proc_type2labelnew = {{PROC_IMPORT, PROC_IMPORT_LABELNEW}, @@ -781,7 +792,9 @@ static std::map proc_type2labelnew = {{PROC_IMPORT, PROC_IMPOR {PROC_TOMO_CTFREFINE, PROC_TOMO_CTFREFINE_LABELNEW}, {PROC_TOMO_ALIGN, PROC_TOMO_ALIGN_LABELNEW}, {PROC_TOMO_RECONSTRUCT, PROC_TOMO_RECONSTRUCT_LABELNEW}, - {PROC_EXTERNAL, PROC_EXTERNAL_LABELNEW}}; + {PROC_TOMO_ALIGN_TILTSERIES, PROC_TOMO_ALIGN_TILTSERIES_LABELNEW}, + {PROC_TOMO_RECONSTRUCT_TOMOGRAM, PROC_TOMO_RECONSTRUCT_TOMOGRAM_LABELNEW}, + {PROC_EXTERNAL, PROC_EXTERNAL_LABELNEW}}; static std::map proc_dirname2type = { {PROC_IMPORT_DIRNAME, PROC_IMPORT}, @@ -808,7 +821,9 @@ static std::map proc_dirname2type = { {PROC_TOMO_CTFREFINE_DIRNAME, PROC_TOMO_CTFREFINE}, {PROC_TOMO_ALIGN_DIRNAME, PROC_TOMO_ALIGN}, {PROC_TOMO_RECONSTRUCT_DIRNAME, PROC_TOMO_RECONSTRUCT}, - {PROC_EXTERNAL_DIRNAME, PROC_EXTERNAL}}; + {PROC_TOMO_ALIGN_TILTSERIES_DIRNAME, PROC_TOMO_ALIGN_TILTSERIES}, + {PROC_TOMO_RECONSTRUCT_TOMOGRAM_DIRNAME, PROC_TOMO_RECONSTRUCT_TOMOGRAM}, + {PROC_EXTERNAL_DIRNAME, PROC_EXTERNAL}}; static std::map proc_labelnew2type = { {PROC_IMPORT_LABELNEW, PROC_IMPORT}, @@ -835,7 +850,9 @@ static std::map proc_labelnew2type = { {PROC_TOMO_CTFREFINE_LABELNEW, PROC_TOMO_CTFREFINE}, {PROC_TOMO_ALIGN_LABELNEW, PROC_TOMO_ALIGN}, {PROC_TOMO_RECONSTRUCT_LABELNEW, PROC_TOMO_RECONSTRUCT}, - {PROC_EXTERNAL_LABELNEW, PROC_EXTERNAL}}; + {PROC_TOMO_ALIGN_TILTSERIES_LABELNEW, PROC_TOMO_ALIGN_TILTSERIES}, + {PROC_TOMO_RECONSTRUCT_TOMOGRAM_LABELNEW, PROC_TOMO_RECONSTRUCT_TOMOGRAM}, + {PROC_EXTERNAL_LABELNEW, PROC_EXTERNAL}}; static std::string get_proc_label(int type) @@ -1227,7 +1244,16 @@ class RelionJob bool getCommandsTomoImportJob(std::string &outputname, std::vector &commands, std::string &final_command, bool do_makedir, int job_counter, std::string &error_message); - void initialiseTomoSubtomoJob(); + void initialiseTomoAlignTiltSeriesJob(); + bool getCommandsTomoAlignTiltSeriesJob(std::string &outputname, std::vector &commands, + std::string &final_command, bool do_makedir, int job_counter, std::string &error_message); + + void initialiseTomoReconstructTomogramsJob(); + bool getCommandsTomoReconstructTomogramsJob(std::string &outputname, std::vector &commands, + std::string &final_command, bool do_makedir, int job_counter, std::string &error_message); + + + void initialiseTomoSubtomoJob(); bool getCommandsTomoSubtomoJob(std::string &outputname, std::vector &commands, std::string &final_command, bool do_makedir, int job_counter, std::string &error_message); From 23e3cecca34b07b9c625584e5e5422ce9fda82ca Mon Sep 17 00:00:00 2001 From: scheres Date: Thu, 21 Apr 2022 11:55:07 +0100 Subject: [PATCH 017/495] add wrapper to run alisters IMOD wrapper in parallel from the GUI --- src/align_tiltseries_runner.cpp | 237 ++++++++++++++++++++++++++ src/align_tiltseries_runner.h | 107 ++++++++++++ src/align_tiltseries_runner_mpi.cpp | 76 +++++++++ src/align_tiltseries_runner_mpi.h | 49 ++++++ src/apps/run_align_tiltseries.cpp | 41 +++++ src/apps/run_align_tiltseries_mpi.cpp | 44 +++++ src/gui_jobwindow.cpp | 4 + src/pipeline_jobs.cpp | 38 +++-- 8 files changed, 581 insertions(+), 15 deletions(-) create mode 100644 src/align_tiltseries_runner.cpp create mode 100644 src/align_tiltseries_runner.h create mode 100644 src/align_tiltseries_runner_mpi.cpp create mode 100644 src/align_tiltseries_runner_mpi.h create mode 100644 src/apps/run_align_tiltseries.cpp create mode 100644 src/apps/run_align_tiltseries_mpi.cpp diff --git a/src/align_tiltseries_runner.cpp b/src/align_tiltseries_runner.cpp new file mode 100644 index 000000000..459b5ba3b --- /dev/null +++ b/src/align_tiltseries_runner.cpp @@ -0,0 +1,237 @@ +/*************************************************************************** + * + * Author: "Sjors H.W. Scheres" + * MRC Laboratory of Molecular Biology + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * This complete copyright notice must be included in any revised version of the + * source code. Additional authorship citations may be added, but existing + * author citations must be preserved. + ***************************************************************************/ +#include "src/align_tiltseries_runner.h" + +void AlignTiltseriesRunner::read(int argc, char **argv, int rank) +{ + parser.setCommandLine(argc, argv); + int gen_section = parser.addSection("General options"); + fn_in = parser.getOption("--i", "STAR file with all input tomograms, or a unix wildcard to all tomogram files, e.g. \"mics/*.mrc\""); + fn_out = parser.getOption("--o", "Directory, where all output files will be stored", "AlignTiltSeries/"); + continue_old = parser.checkOption("--only_do_unfinished", "Only estimate CTFs for those tomograms for which there is not yet a logfile with Final values."); + do_at_most = textToInteger(parser.getOption("--do_at_most", "Only process up to this number of (unprocessed) tomograms.", "-1")); + + int fid_section = parser.addSection("IMOD fiducial-based alignment options"); + do_imod_fiducials = parser.checkOption("--imod_fiducials", "Use IMOD's fiducial-based alignment method"); + fiducial_diam = textToFloat(parser.getOption("--fiducial_diameter", "Diameter of the fiducials (in nm)", "10")); + + int pat_section = parser.addSection("IMOD patch-tracking alignment options"); + do_imod_patchtrack = parser.checkOption("--imod_patchtrack", "OR: Use IMOD's patrick-tracking alignment method"); + patch_overlap = textToFloat(parser.getOption("--patch_overlap", "Overlap between the patches (in %)", "10")); + patch_size = textToInteger(parser.getOption("--patch_size", "Patch size (in unbinned pixels)", "10")); + + int exp_section = parser.addSection("Expert options"); + other_wrapper_args = parser.checkOption("--other_wrapper_args", "Additional command-line arguments that will be passed onto the wrapper."); + + // Initialise verb for non-parallel execution + verb = 1; + + // Check for errors in the command-line option + if (parser.checkForErrors()) + REPORT_ERROR("Errors encountered on the command line (see above), exiting..."); +} + +void AlignTiltseriesRunner::usage() +{ + parser.writeUsage(std::cout); +} + +void AlignTiltseriesRunner::initialise(bool is_leader) +{ + // Get the Imod wrapper executable + if (fn_imodwrapper_exe == "") + { + char *penv; + penv = getenv("RELION_IMOD_WRAPPER_EXECUTABLE"); + if (penv != NULL) + fn_imodwrapper_exe = (std::string)penv; + } + + int i = 0; + if (do_imod_fiducials) i++; + if (do_imod_patchtrack) i++; + if (i != 1) REPORT_ERROR("ERROR: you need to specify one of these options: --imod_fiducials or --imod_patchtrack"); + + // Make sure fn_out ends with a slash + if (fn_out[fn_out.length()-1] != '/') + fn_out += "/"; + + // Check if this is a TomographyExperiment starfile, if not raise an error + if (!tomogramSet.read(fn_in, 1)) + { + REPORT_ERROR("ERROR: the input file is not a valid tomograms.star file"); + } + + idx_tomograms_all.clear(); + idx_tomograms.clear(); + bool warned = false; + for (long int itomo = 0; itomo < tomogramSet.size(); itomo++) + { + FileName fn_star; + tomogramSet.globalTable.getValue(EMDL_TOMO_TILT_SERIES_STARFILE, fn_star, itomo); + FileName fn_newstar = getOutputFileWithNewUniqueDate(fn_star, fn_out); + tomogramSet.globalTable.setValue(EMDL_TOMO_TILT_SERIES_STARFILE, fn_newstar, itomo); + + bool process_this = true; + bool ignore_this = false; + if (continue_old) + { + if (checkImodWrapperResults(itomo)) + { + process_this = false; // already done + } + } + + if (do_at_most >= 0 && idx_tomograms.size() >= do_at_most) + { + if (process_this) { + ignore_this = true; + process_this = false; + if (!warned) + { + warned = true; + std::cout << "NOTE: processing of some tomograms will be skipped as requested by --do_at_most" << std::endl; + } + } + // If this tomogram has already been processed, the result should be included in the output. + // So ignore_this remains false. + } + + if (process_this) + { + idx_tomograms.push_back(itomo); + } + + if (!ignore_this) + { + idx_tomograms_all.push_back(itomo); + } + } + + if (is_leader && do_at_most >= 0 ) + { + std::cout << tomogramSet.size() << " tomograms were given in the input tomogram set, but we process only "; + std::cout << do_at_most << " tomograms as specified in --do_at_most." << std::endl; + } + + if (verb > 0) + { + std::cout << " Using IMOD wrapper executable in: " << fn_imodwrapper_exe << std::endl; + std::cout << " to align tilt series for the following tomograms: " << std::endl; + if (continue_old) + std::cout << " (skipping all tomograms for which a logfile with Final values already exists " << std::endl; + for (unsigned int i = 0; i < idx_tomograms.size(); ++i) + std::cout << " * " << tomogramSet.getTomogramName(idx_tomograms[i]) << std::endl; + } +} + +void AlignTiltseriesRunner::run() +{ + + int barstep; + if (verb > 0) + { + std::cout << " Aligning tilt series ..." << std::endl; + init_progress_bar(idx_tomograms.size()); + barstep = XMIPP_MAX(1, idx_tomograms.size() / 60); + } + + std::vector alltomonames; + for (long int itomo = 0; itomo < idx_tomograms.size(); itomo++) + { + + // Abort through the pipeline_control system + if (pipeline_control_check_abort_job()) + exit(RELION_EXIT_ABORTED); + + executeImodWrapper(idx_tomograms[itomo]); + + if (verb > 0 && itomo % barstep == 0) + progress_bar(itomo); + } + + if (verb > 0) + progress_bar(idx_tomograms.size()); + + joinImodWrapperResults(); +} + + +void AlignTiltseriesRunner::executeImodWrapper(long idx_tomo) +{ + + std::string command = "relion_tomo_align_tilt_series "; + command += " --tilt-series-star-file " + fn_in; + command += " --tomogram-name " + tomogramSet.getTomogramName(idx_tomo); + command += " --output-directory " + fn_out; + // Make sure the methods are the first argument to the program! + if (do_imod_fiducials) + { + command += " IMOD:fiducials"; + command += " --nominal-fiducial-diameter-nanometres " + floatToString(fiducial_diam); + } + else if (do_imod_patchtrack) + { + command += " IMOD:patch-tracking"; + command += " --unbinned-patch-size-pixels " + integerToString(patch_size); + command += " -- patch-overlap-percentage " + floatToString(patch_overlap); + } + command += other_wrapper_args; + + if (system(command.c_str())) + std::cerr << "WARNING: there was an error in executing: " << command << std::endl; + +} + +bool AlignTiltseriesRunner::checkImodWrapperResults(long idx_tomo) +{ + FileName fn_star; + tomogramSet.globalTable.getValue(EMDL_TOMO_TILT_SERIES_STARFILE, fn_star, idx_tomo); + + MetaDataTable MDtomo; + MDtomo.read(fn_star); + return (MDtomo.containsLabel(EMDL_ORIENT_ORIGIN_X_ANGSTROM) && + MDtomo.containsLabel(EMDL_ORIENT_ORIGIN_Y_ANGSTROM) && + MDtomo.containsLabel(EMDL_ORIENT_ROT) && + MDtomo.containsLabel(EMDL_ORIENT_TILT) && + MDtomo.containsLabel(EMDL_ORIENT_PSI) ); + +} + +void AlignTiltseriesRunner::joinImodWrapperResults() +{ + // Check again the STAR file exists and has the right labels + for (long itomo = 0; itomo < tomogramSet.size(); itomo++) + { + if (!checkImodWrapperResults(itomo)) + { + if (verb) std::cerr << "WARNING: cannot find tilt series alignment parameters in " << tomogramSet.getTomogramName(itomo) << std::endl; + } + + } + + tomogramSet.globalTable.write(fn_out + "aligned_tilt_series.star"); + + if (verb > 0) + { + std::cout << " Done! Written out: " << fn_out << "aligned_tilt_series.star" << std::endl; + } + +} diff --git a/src/align_tiltseries_runner.h b/src/align_tiltseries_runner.h new file mode 100644 index 000000000..68f4f7381 --- /dev/null +++ b/src/align_tiltseries_runner.h @@ -0,0 +1,107 @@ +/*************************************************************************** + * + * Author: "Sjors H.W. Scheres" + * MRC Laboratory of Molecular Biology + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * This complete copyright notice must be included in any revised version of the + * source code. Additional authorship citations may be added, but existing + * author citations must be preserved. + ***************************************************************************/ +#ifndef RELION_ALIGN_TILTSERIES_RUNNER_H +#define RELION_ALIGN_TILTSERIES_RUNNER_H + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "src/jaz/tomography/tomogram_set.h" + +class AlignTiltseriesRunner +{ +public: + + // I/O Parser + IOParser parser; + + // Verbosity + int verb; + + // Output rootname + FileName fn_in, fn_out; + + // Filenames of all the tomograms to estimate the CTF from + std::vector idx_tomograms, idx_tomograms_all; + + // Information about tomography experiment + TomogramSet tomogramSet; + + // CTFFIND and Gctf executables and shell + FileName fn_imodwrapper_exe, fn_shell; + + // Use IMOD:fiducials + bool do_imod_fiducials; + + // Use IMOD:patch-tracking + bool do_imod_patchtrack; + + // Nominal fiducial diameter (nm) + RFLOAT fiducial_diam; + + // Unbinned patch size (pixels) + int patch_size; + + // Patch overlap (percentage 0-100) + RFLOAT patch_overlap; + + // Additional gctf command line options + std::string other_wrapper_args; + + // Continue an old run: only estimate CTF if logfile WITH Final Values line does not yet exist, otherwise skip the tomogram + bool continue_old; + + // Process at most this number of unprocessed tomograms + long do_at_most; + +public: + // Read command line arguments + void read(int argc, char **argv, int rank = 0); + + // Print usage instructions + void usage(); + + // Initialise some stuff after reading + void initialise(bool is_leader = true); + + // Execute all CTFFIND jobs to get CTF parameters + void run(); + + // Check STAR file for tomogram exists and has the correct labels + bool checkImodWrapperResults(long idx_tomo); + + // Execute CTFFIND for a single tomogram + void executeImodWrapper(long idx_tomo); + + // Harvest all IMOD results into the single tomograms set starfile, and write it out + void joinImodWrapperResults(); + +}; + + +#endif //RELION_ALIGN_TILTSERIES_RUNNER_H diff --git a/src/align_tiltseries_runner_mpi.cpp b/src/align_tiltseries_runner_mpi.cpp new file mode 100644 index 000000000..231f5ad37 --- /dev/null +++ b/src/align_tiltseries_runner_mpi.cpp @@ -0,0 +1,76 @@ +/*************************************************************************** + * + * Author: "Sjors H.W. Scheres" + * MRC Laboratory of Molecular Biology + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * This complete copyright notice must be included in any revised version of the + * source code. Additional authorship citations may be added, but existing + * author citations must be preserved. + ***************************************************************************/ +#include "src/align_tiltseries_runner_mpi.h" + +void AlignTiltseriesRunnerMpi::read(int argc, char **argv) +{ + // Define a new MpiNode + node = new MpiNode(argc, argv); + + // First read in non-parallelisation-dependent variables + AlignTiltseriesRunner::read(argc, argv); + + // Don't put any output to screen for mpi followers + verb = (node->isLeader()) ? 1 : 0; + + // Possibly also read parallelisation-dependent variables here + + // Print out MPI info + printMpiNodesMachineNames(*node); +} + +void AlignTiltseriesRunnerMpi::run() +{ + + // Each node does part of the work + long int my_first_tomogram, my_last_tomogram, my_nr_tomograms; + divide_equally(idx_tomograms.size(), node->size, node->rank, my_first_tomogram, my_last_tomogram); + my_nr_tomograms = my_last_tomogram - my_first_tomogram + 1; + + int barstep; + if (verb > 0) + { + std::cout << " Aligning tilt series ..." << std::endl; + init_progress_bar(my_nr_tomograms); + barstep = XMIPP_MAX(1, my_nr_tomograms / 60); + } + + std::vector allmicnames; + for (long int itomo = my_first_tomogram; itomo <= my_last_tomogram; itomo++) + { + + // Abort through the pipeline_control system + if (pipeline_control_check_abort_job()) + MPI_Abort(MPI_COMM_WORLD, RELION_EXIT_ABORTED); + + executeImodWrapper(idx_tomograms[itomo]); + + if (verb > 0 && itomo % barstep == 0) progress_bar(itomo); + + } + + if (verb > 0) progress_bar(my_nr_tomograms); + + MPI_Barrier(MPI_COMM_WORLD); + + // Only the leader writes the joined result file + if (node->isLeader()) joinImodWrapperResults(); + +} diff --git a/src/align_tiltseries_runner_mpi.h b/src/align_tiltseries_runner_mpi.h new file mode 100644 index 000000000..9f9bcdf7b --- /dev/null +++ b/src/align_tiltseries_runner_mpi.h @@ -0,0 +1,49 @@ +/*************************************************************************** + * + * Author: "Sjors H.W. Scheres" + * MRC Laboratory of Molecular Biology + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * This complete copyright notice must be included in any revised version of the + * source code. Additional authorship citations may be added, but existing + * author citations must be preserved. + ***************************************************************************/ + +#ifndef ALIGN_TILTSERIES_RUNNER_MPI_H_ +#define ALIGN_TILTSERIES_RUNNER_MPI_H_ + +#include "src/mpi.h" +#include "src/align_tiltseries_runner.h" +#include "src/parallel.h" + +class AlignTiltseriesRunnerMpi: public AlignTiltseriesRunner +{ +public: + MpiNode *node; + + /** Destructor, calls MPI_Finalize */ + ~AlignTiltseriesRunnerMpi() + { + delete node; + } + + /** Read + * This could take care of mpi-parallelisation-dependent variables + */ + void read(int argc, char **argv); + + // Parallelized run function + void run(); + +}; + +#endif //ALIGN_TILTSERIES_RUNNER_MPI_H diff --git a/src/apps/run_align_tiltseries.cpp b/src/apps/run_align_tiltseries.cpp new file mode 100644 index 000000000..29b93755b --- /dev/null +++ b/src/apps/run_align_tiltseries.cpp @@ -0,0 +1,41 @@ +/*************************************************************************** + * + * Author: "Sjors H.W. Scheres" + * MRC Laboratory of Molecular Biology + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * This complete copyright notice must be included in any revised version of the + * source code. Additional authorship citations may be added, but existing + * author citations must be preserved. + ***************************************************************************/ +#include + + +int main(int argc, char *argv[]) +{ + AlignTiltseriesRunner prm; + + try + { + prm.read(argc, argv); + prm.initialise(); + prm.run(); + } + catch (RelionError XE) + { + //prm.usage(); + std::cerr << XE; + return RELION_EXIT_FAILURE; + } + + return RELION_EXIT_SUCCESS; +} diff --git a/src/apps/run_align_tiltseries_mpi.cpp b/src/apps/run_align_tiltseries_mpi.cpp new file mode 100644 index 000000000..88fbb43b6 --- /dev/null +++ b/src/apps/run_align_tiltseries_mpi.cpp @@ -0,0 +1,44 @@ +/*************************************************************************** + * + * Author: "Sjors H.W. Scheres" + * MRC Laboratory of Molecular Biology + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * This complete copyright notice must be included in any revised version of the + * source code. Additional authorship citations may be added, but existing + * author citations must be preserved. + ***************************************************************************/ +#include + + +int main(int argc, char *argv[]) +{ + AlignTiltseriesRunnerMpi prm; + + try + { + prm.read(argc, argv); + prm.initialise(prm.node->isLeader()); + MPI_Barrier(MPI_COMM_WORLD); + prm.run(); + } + catch (RelionError XE) + { + if (prm.verb > 0) + //prm.usage(); + std::cerr << XE; + MPI_Abort(MPI_COMM_WORLD, RELION_EXIT_FAILURE); + } + + MPI_Barrier(MPI_COMM_WORLD); + return RELION_EXIT_SUCCESS; +} diff --git a/src/gui_jobwindow.cpp b/src/gui_jobwindow.cpp index bf2e8e6cb..804032e54 100644 --- a/src/gui_jobwindow.cpp +++ b/src/gui_jobwindow.cpp @@ -2504,6 +2504,10 @@ void JobWindow::initialiseTomoAlignTiltseriesWindow() group2->end(); guientries["do_imod_patchtrack"].cb_menu_i(); // make default active + // Add a little spacer + current_y += STEPY/2; + place("other_wrapper_args", TOGGLE_DEACTIVATE); + tab1->end(); diff --git a/src/pipeline_jobs.cpp b/src/pipeline_jobs.cpp index ce7568e7b..03afe9f2d 100644 --- a/src/pipeline_jobs.cpp +++ b/src/pipeline_jobs.cpp @@ -6358,6 +6358,8 @@ void RelionJob::initialiseTomoAlignTiltSeriesJob() joboptions["patch_size"] = JobOption("Patch size (in unbinned pixels): ", 10, 1, 50, 1, "The size of the patches in unbinned pixels."); joboptions["patch_overlap"] = JobOption("Patch overlap (%): ", 10, 0, 100, 10, "The overlap (0-100%) between the patches."); + joboptions["other_wrapper_args"] = JobOption("Other wrapper arguments:", (std::string)"", "Other arguments that will be passed straight onto the IMOD wrapper."); + } bool RelionJob::getCommandsTomoAlignTiltSeriesJob(std::string &outputname, std::vector &commands, std::string &final_command, bool do_makedir, int job_counter, std::string &error_message) @@ -6369,35 +6371,41 @@ bool RelionJob::getCommandsTomoAlignTiltSeriesJob(std::string &outputname, std:: int i = 0; if (joboptions["do_imod_fiducials"].getBoolean()) i++; if (joboptions["do_imod_patchtrack"].getBoolean()) i++; - if (i != 1) { error_message = "ERROR: you should (only) select ONE of the alignment methods: IMOD:fiducials or IMOD:patchtracking."; return false; } - // TODO: rename command to relion_tomo_import_tiltseries? - command = "relion_tomo_align_tilt_series "; + if (joboptions["nr_mpi"].getNumber(error_message) > 1) + command="`which relion_run_align_tiltseries_mpi`"; + else + command="`which relion_run_align_tiltseries`"; + if (error_message != "") return false; + + command += " --i " + joboptions["in_tiltseries"].getString(); + Node node(joboptions["in_tiltseries"].getString(), joboptions["in_tiltseries"].node_type); + inputNodes.push_back(node); + + command += " --o " + outputname; + Node node2(outputname+"aligned_tilt_series.star", LABEL_TOMO_TOMOGRAMS); + outputNodes.push_back(node2); + // Make sure the methods are the first argument to the program! if (joboptions["do_imod_fiducials"].getBoolean()) { - command += " IMOD:fiducials"; - command += " --nominal-fiducial-diameter-nanometres " + joboptions["fiducial_diameter"].getString(); + command += " --imod_fiducials"; + command += " --fiducial_diameter " + joboptions["fiducial_diameter"].getString(); } else if (joboptions["do_imod_patchtrack"].getBoolean()) { - command += " IMOD:patch-tracking"; - command += " --unbinned-patch-size-pixels " + joboptions["patch_size"].getString(); - command += " -- patch-overlap-percentage " + joboptions["patch_overlap"].getString(); + command += " --imod_patchtrack"; + command += " --patch_size " + joboptions["patch_size"].getString(); + command += " -- patch_overlap " + joboptions["patch_overlap"].getString(); } - command += " --tilt-series-star-file " + joboptions["in_tiltseries"].getString(); - Node node(joboptions["in_tiltseries"].getString(), joboptions["in_tiltseries"].node_type); - inputNodes.push_back(node); - - command += " --output-directory " + outputname; - Node node2(outputname+"tilt_series.star", LABEL_TOMO_TOMOGRAMS); - outputNodes.push_back(node2); + if ((joboptions["other_wrapper_args"].getString()).length() > 0) + command += " --other_wrapper_args \" " + joboptions["other_wrapper_args"].getString() + " \""; // Other arguments for extraction command += " " + joboptions["other_args"].getString(); From 9175cc2f98a84f7cb3fcd00613451f1a2c6b4bde Mon Sep 17 00:00:00 2001 From: scheres Date: Thu, 21 Apr 2022 15:51:25 +0100 Subject: [PATCH 018/495] bug fixing tiltseries alignment through IMOD wrapper --- src/align_tiltseries_runner.cpp | 10 ++++++---- src/mask.cpp | 9 +++++++-- src/pipeline_jobs.cpp | 15 ++++++++------- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/src/align_tiltseries_runner.cpp b/src/align_tiltseries_runner.cpp index 459b5ba3b..60ca98c80 100644 --- a/src/align_tiltseries_runner.cpp +++ b/src/align_tiltseries_runner.cpp @@ -178,10 +178,7 @@ void AlignTiltseriesRunner::executeImodWrapper(long idx_tomo) { std::string command = "relion_tomo_align_tilt_series "; - command += " --tilt-series-star-file " + fn_in; - command += " --tomogram-name " + tomogramSet.getTomogramName(idx_tomo); - command += " --output-directory " + fn_out; - // Make sure the methods are the first argument to the program! + // Make sure the methods are the first argument to the program! if (do_imod_fiducials) { command += " IMOD:fiducials"; @@ -193,6 +190,11 @@ void AlignTiltseriesRunner::executeImodWrapper(long idx_tomo) command += " --unbinned-patch-size-pixels " + integerToString(patch_size); command += " -- patch-overlap-percentage " + floatToString(patch_overlap); } + + command += " --tilt-series-star-file " + fn_in; + command += " --tomogram-name " + tomogramSet.getTomogramName(idx_tomo); + command += " --output-directory " + fn_out; + command += other_wrapper_args; if (system(command.c_str())) diff --git a/src/mask.cpp b/src/mask.cpp index cc3899c3a..5b273aa4e 100644 --- a/src/mask.cpp +++ b/src/mask.cpp @@ -138,8 +138,13 @@ void softMaskOutsideMapForHelix( if ( (cosine_width < 0.) || (mask_sphere_radius_pix < 1.) || (mask_sphere_radius_pix > boxsize) || (mask_cyl_radius_pix < 1.) || (mask_cyl_radius_pix > boxsize) - || (mask_sphere_radius_pix < mask_cyl_radius_pix) ) - REPORT_ERROR("mask.cpp::softMaskOutsideMapForHelix(): Invalid radii of spherical and cylindrical masks or soft cosine widths!"); + || (mask_sphere_radius_pix < mask_cyl_radius_pix) ) { + std::cerr << " cosine_width= " << cosine_width << std::endl; + std::cerr << " boxsize= " << boxsize << std::endl; + std::cerr << " mask_sphere_radius_pix= " << mask_sphere_radius_pix << std::endl; + std::cerr << " mask_cyl_radius_pix= " << mask_cyl_radius_pix << std::endl; + REPORT_ERROR("mask.cpp::softMaskOutsideMapForHelix(): Invalid radii of spherical and cylindrical masks or soft cosine widths! Is boxsize > mask_sphere_radius > mask_cyl_radius?"); + } // Spherical mask: 0 < R1 < R2 R1 = mask_sphere_radius_pix; diff --git a/src/pipeline_jobs.cpp b/src/pipeline_jobs.cpp index 03afe9f2d..ea636a07a 100644 --- a/src/pipeline_jobs.cpp +++ b/src/pipeline_jobs.cpp @@ -6383,13 +6383,6 @@ bool RelionJob::getCommandsTomoAlignTiltSeriesJob(std::string &outputname, std:: command="`which relion_run_align_tiltseries`"; if (error_message != "") return false; - command += " --i " + joboptions["in_tiltseries"].getString(); - Node node(joboptions["in_tiltseries"].getString(), joboptions["in_tiltseries"].node_type); - inputNodes.push_back(node); - - command += " --o " + outputname; - Node node2(outputname+"aligned_tilt_series.star", LABEL_TOMO_TOMOGRAMS); - outputNodes.push_back(node2); // Make sure the methods are the first argument to the program! if (joboptions["do_imod_fiducials"].getBoolean()) @@ -6404,6 +6397,14 @@ bool RelionJob::getCommandsTomoAlignTiltSeriesJob(std::string &outputname, std:: command += " -- patch_overlap " + joboptions["patch_overlap"].getString(); } + command += " --i " + joboptions["in_tiltseries"].getString(); + Node node(joboptions["in_tiltseries"].getString(), joboptions["in_tiltseries"].node_type); + inputNodes.push_back(node); + + command += " --o " + outputname; + Node node2(outputname+"aligned_tilt_series.star", LABEL_TOMO_TOMOGRAMS); + outputNodes.push_back(node2); + if ((joboptions["other_wrapper_args"].getString()).length() > 0) command += " --other_wrapper_args \" " + joboptions["other_wrapper_args"].getString() + " \""; From 6559a0e748fa5bec465988f5abed07e5603bd035 Mon Sep 17 00:00:00 2001 From: scheres Date: Tue, 26 Apr 2022 12:36:34 +0100 Subject: [PATCH 019/495] untested modification to tomogramSet to read from original motion corrected micrographs, instead of a pre-made tilt series stack --- src/jaz/tomography/apps/import_tomograms.cpp | 12 +-- src/jaz/tomography/tomogram_set.cpp | 98 ++++++++++++++------ src/jaz/tomography/tomogram_set.h | 2 +- 3 files changed, 77 insertions(+), 35 deletions(-) diff --git a/src/jaz/tomography/apps/import_tomograms.cpp b/src/jaz/tomography/apps/import_tomograms.cpp index acd5b4333..f85bc8a85 100644 --- a/src/jaz/tomography/apps/import_tomograms.cpp +++ b/src/jaz/tomography/apps/import_tomograms.cpp @@ -381,12 +381,12 @@ int main(int argc, char *argv[]) ctfs[i] = ctfs[mapping.oldFrameIndex[i]]; } - tomograms.addTomogram( - name, tsFn, - mapping.projections, - mapping.w, mapping.h, mapping.d, - cumulativeDose, fractionalDose, - ctfs, hand, pixelSize); + tomograms.addTomogramFromIMODStack( + name, tsFn, + mapping.projections, + mapping.w, mapping.h, mapping.d, + cumulativeDose, fractionalDose, + ctfs, hand, pixelSize); Log::endSection(); } diff --git a/src/jaz/tomography/tomogram_set.cpp b/src/jaz/tomography/tomogram_set.cpp index 8b6865ec5..0e66f6a53 100644 --- a/src/jaz/tomography/tomogram_set.cpp +++ b/src/jaz/tomography/tomogram_set.cpp @@ -154,42 +154,86 @@ Tomogram TomogramSet::loadTomogram(int index, bool loadImageData) const { Tomogram out; - std::string tomoName, stackFn; + std::string tomoName; + i3Vector stackSize; globalTable.getValueSafely(EMDL_TOMO_NAME, tomoName, index); - globalTable.getValueSafely(EMDL_TOMO_TILT_SERIES_NAME, stackFn, index); - globalTable.getValueSafely(EMDL_TOMO_FRAME_COUNT, out.frameCount, index); + const MetaDataTable& m = tomogramTables[index]; - i3Vector stackSize; + globalTable.getValueSafely(EMDL_TOMO_SIZE_X, out.w0, index); + globalTable.getValueSafely(EMDL_TOMO_SIZE_Y, out.h0, index); + globalTable.getValueSafely(EMDL_TOMO_SIZE_Z, out.d0, index); - if (loadImageData) - { - out.stack.read(stackFn); - out.hasImage = true; + if (globalTable.containsLabel(EMDL_TOMO_TILT_SERIES_NAME)) + { + // option A: Kino's original IMOD import functionality - stackSize.x = out.stack.xdim; - stackSize.y = out.stack.ydim; - stackSize.z = out.stack.zdim; - } - else - { - out.hasImage = false; + globalTable.getValueSafely(EMDL_TOMO_TILT_SERIES_NAME, out.tiltSeriesFilename, index); + globalTable.getValueSafely(EMDL_TOMO_FRAME_COUNT, out.frameCount, index); - t3Vector isl = ImageFileHelper::getSize(stackFn); + if (loadImageData) + { + out.stack.read(out.tiltSeriesFilename); + out.hasImage = true; - stackSize.x = isl.x; - stackSize.y = isl.y; - stackSize.z = isl.z; - } + stackSize.x = out.stack.xdim; + stackSize.y = out.stack.ydim; + stackSize.z = out.stack.zdim; + } + else + { + out.hasImage = false; - out.imageSize = stackSize.xy(); + t3Vector isl = ImageFileHelper::getSize(out.tiltSeriesFilename); + + stackSize.x = isl.x; + stackSize.y = isl.y; + stackSize.z = isl.z; + } + + } + else + { + // option B: new functionality to work directly with images from RELION's motioncorr runner - out.tiltSeriesFilename = stackFn; + out.tiltSeriesFilename = ""; + out.frameCount = m.numberOfObjects(); - globalTable.getValueSafely(EMDL_TOMO_SIZE_X, out.w0, index); - globalTable.getValueSafely(EMDL_TOMO_SIZE_Y, out.h0, index); - globalTable.getValueSafely(EMDL_TOMO_SIZE_Z, out.d0, index); + // Get image size from the first image in the tomogramTable + std::string fn_img; + m.getValueSafely(EMDL_MICROGRAPH_NAME, fn_img, 0); + Image I; + I.read(fn_img,false); // false means don't read the actual image data, only the header + stackSize.x = XSIZE(I()); + stackSize.y = YSIZE(I()); + stackSize.z = out.frameCount; + + if (loadImageData) + { + Image myStack(stackSize.x, stackSize.y, stackSize.z); + for (int f = 0; f < out.frameCount; f++) + { + m.getValueSafely(EMDL_MICROGRAPH_NAME, fn_img, f); + Image I2; + I2.read(fn_img); + + if (XSIZE(I2()) != stackSize.x || YSIZE(I2()) != stackSize.y) + { + REPORT_ERROR("ERROR: unequal image dimensions in the individual tilt series images of tomogram: " + tomoName); + } + myStack().setSlice(f, I2()); + } + out.stack.copyDataAndSizeFrom(myStack); + } + else + { + out.hasImage = false; + } + + } + + out.imageSize = stackSize.xy(); out.centre = d3Vector(out.w0/2.0, out.h0/2.0, out.d0/2.0); globalTable.getValueSafely(EMDL_TOMO_HANDEDNESS, out.handedness, index); @@ -224,8 +268,6 @@ Tomogram TomogramSet::loadTomogram(int index, bool loadImageData) const } } - const MetaDataTable& m = tomogramTables[index]; - out.cumulativeDose.resize(out.frameCount); out.centralCTFs.resize(out.frameCount); out.projectionMatrices.resize(out.frameCount); @@ -328,7 +370,7 @@ Tomogram TomogramSet::loadTomogram(int index, bool loadImageData) const return out; } -void TomogramSet::addTomogram( +void TomogramSet::addTomogramFromIMODStack( std::string tomoName, std::string stackFilename, const std::vector& projections, int w, int h, int d, diff --git a/src/jaz/tomography/tomogram_set.h b/src/jaz/tomography/tomogram_set.h index b01dea584..073268797 100644 --- a/src/jaz/tomography/tomogram_set.h +++ b/src/jaz/tomography/tomogram_set.h @@ -25,7 +25,7 @@ class TomogramSet Tomogram loadTomogram(int index, bool loadImageData) const; - void addTomogram( + void addTomogramFromIMODStack( std::string tomoName, std::string stackFilename, const std::vector& projections, int w, int h, int d, From 89d23f81a2905956e115561acbd9f9963935398e Mon Sep 17 00:00:00 2001 From: scheres Date: Tue, 26 Apr 2022 12:47:14 +0100 Subject: [PATCH 020/495] just check rlnMicrographName is present in tomogramTable --- src/jaz/tomography/tomogram_set.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/jaz/tomography/tomogram_set.cpp b/src/jaz/tomography/tomogram_set.cpp index 0e66f6a53..cee3c0673 100644 --- a/src/jaz/tomography/tomogram_set.cpp +++ b/src/jaz/tomography/tomogram_set.cpp @@ -200,6 +200,11 @@ Tomogram TomogramSet::loadTomogram(int index, bool loadImageData) const out.frameCount = m.numberOfObjects(); // Get image size from the first image in the tomogramTable + if (!m.containsLabel(EMDL_MICROGRAPH_NAME)) + { + REPORT_ERROR("ERROR: tomogramTable for " + tomoName + " does not contain a rlnMicrographName label, yet the globalTable also does not contain a rlnTomoTiltSeriesName label!"); + } + std::string fn_img; m.getValueSafely(EMDL_MICROGRAPH_NAME, fn_img, 0); Image I; From 62fc8f0854a886616320fa7391431d9874484ea2 Mon Sep 17 00:00:00 2001 From: scheres Date: Wed, 27 Apr 2022 15:05:05 +0100 Subject: [PATCH 021/495] compulsory dimensions for tomo_reconstruct_tomogram --- .../programs/reconstruct_tomogram.cpp | 59 ++++++++++--------- .../programs/reconstruct_tomogram.h | 2 +- src/jaz/tomography/tomogram_set.cpp | 17 ++++-- src/metadata_label.h | 9 +++ 4 files changed, 53 insertions(+), 34 deletions(-) diff --git a/src/jaz/tomography/programs/reconstruct_tomogram.cpp b/src/jaz/tomography/programs/reconstruct_tomogram.cpp index 74ba01cd2..1ce5f670b 100644 --- a/src/jaz/tomography/programs/reconstruct_tomogram.cpp +++ b/src/jaz/tomography/programs/reconstruct_tomogram.cpp @@ -33,6 +33,11 @@ void TomoBackprojectProgram::readParameters(int argc, char *argv[]) int gen_section = parser.addSection("General options"); tomoName = parser.getOption("--tn", "Tomogram name", "*"); + outFn = parser.getOption("--o", "Output filename (or output directory in case of reconstructing multiple tomograms)"); + + w = textToInteger(parser.getOption("--w", "Width")); + h = textToInteger(parser.getOption("--h", "Height" )); + d = textToInteger(parser.getOption("--d", "Thickness")); applyWeight = !parser.checkOption("--no_weight", "Do not perform weighting in Fourier space using a Wiener filter"); applyPreWeight = parser.checkOption("--pre_weight", "Pre-weight the 2D slices prior to backprojection"); @@ -51,15 +56,10 @@ void TomoBackprojectProgram::readParameters(int argc, char *argv[]) y0 = textToDouble(parser.getOption("--y0", "Y origin", "1.0")); z0 = textToDouble(parser.getOption("--z0", "Z origin", "1.0")); - w = textToInteger(parser.getOption("--w", "Width", "-1.0")); - h = textToInteger(parser.getOption("--h", "Height", "-1.0")); - d = textToInteger(parser.getOption("--d", "Thickness", "-1.0")); - spacing = textToDouble(parser.getOption("--bin", "Binning", "8.0")); n_threads = textToInteger(parser.getOption("--j", "Number of threads", "1")); - outFn = parser.getOption("--o", "Output filename (or directory in case of reconstructing all tomograms)"); Log::readParams(parser); @@ -88,19 +88,33 @@ void TomoBackprojectProgram::initialise() if (tomoName == "*") { + do_multiple = true; + for (int idx = 0; idx < tomogramSet.size(); idx++) { if (do_only_unfinished && exists(getOutputFileName(idx))) continue; tomoIndexTodo.push_back(idx); } + + if (outFn[outFn.size()-1] != '/') outFn += '/'; + FileName fn_dir = outFn + "tomograms/"; + if (!exists(fn_dir)) mktree(fn_dir); + } else { tomoIndexTodo.push_back(tomogramSet.getTomogramIndex(tomoName)); + do_multiple = false; + + if (!exists(outFn.beforeLastOf("/"))) mktree(outFn.beforeLastOf("/")); } - std::cout << " + Reconstructing " << tomoIndexTodo.size() << " tomograms ... " << std::endl; + std::cout << " + Reconstructing " << tomoIndexTodo.size() << " tomograms: " << std::endl; + for (int idx = 0; idx < tomoIndexTodo.size(); idx++) + { + std::cout << " - " << tomogramSet.getTomogramName(tomoIndexTodo[idx]) << std::endl; + } } @@ -109,7 +123,7 @@ void TomoBackprojectProgram::run(int rank, int size) long my_first_idx, my_last_idx; divide_equally(tomoIndexTodo.size(), size, rank , my_first_idx, my_last_idx); - + std::cout << " + Reconstructing ... " << std::endl; int barstep, nr_todo = my_last_idx-my_first_idx+1; if (rank == 0) { @@ -130,8 +144,8 @@ void TomoBackprojectProgram::run(int rank, int size) progress_bar(nr_todo); - // If the output filename was a directory, then also write updated tomograms.star. - if (outFn[outFn.size()-1] == '/') + // If we were doing multiple tomograms, then also write the updated tomograms.star. + if (do_multiple) { tomogramSet.write(outFn + "tomograms.star"); } @@ -144,10 +158,6 @@ void TomoBackprojectProgram::reconstructOneTomogram(int tomoIndex) { Tomogram tomogram = tomogramSet.loadTomogram(tomoIndex, true); - const int w0 = tomogram.w0; - const int h0 = tomogram.h0; - const int d0 = tomogram.d0; - if (zeroDC) Normalization::zeroDC_stack(tomogram.stack); const int fc = tomogram.frameCount; @@ -155,13 +165,7 @@ void TomoBackprojectProgram::reconstructOneTomogram(int tomoIndex) BufferedImage stackAct; std::vector projAct(fc); double pixelSizeAct = tomogram.optics.pixelSize; - - - const int w1 = w > 0? w : w0 / spacing + 0.5; - const int h1 = h > 0? h : h0 / spacing + 0.5; - const int t1 = d > 0? d : d0 / spacing; - if (std::abs(spacing - 1.0) < 1e-2) { projAct = tomogram.projectionMatrices; @@ -177,7 +181,7 @@ void TomoBackprojectProgram::reconstructOneTomogram(int tomoIndex) if (std::abs(spacing - 1.0) > 1e-2) { - Log::print("Resampling image stack"); + if (!do_multiple) Log::print("Resampling image stack"); if (FourierCrop) { @@ -204,7 +208,7 @@ void TomoBackprojectProgram::reconstructOneTomogram(int tomoIndex) d3Vector orig(x0, y0, z0); - BufferedImage out(w1, h1, t1); + BufferedImage out(w, h, d); out.fill(0.f); BufferedImage psfStack; @@ -259,7 +263,7 @@ void TomoBackprojectProgram::reconstructOneTomogram(int tomoIndex) stackAct = RealSpaceBackprojection::preWeight(stackAct, projAct, n_threads); } - Log::print("Backprojecting"); + if (!do_multiple) Log::print("Backprojecting"); RealSpaceBackprojection::backproject( stackAct, projAct, out, n_threads, @@ -268,7 +272,7 @@ void TomoBackprojectProgram::reconstructOneTomogram(int tomoIndex) if (applyWeight || applyCtf) { - BufferedImage psf(w1, h1, t1); + BufferedImage psf(w, h, d); psf.fill(0.f); if (applyCtf) @@ -286,7 +290,7 @@ void TomoBackprojectProgram::reconstructOneTomogram(int tomoIndex) Reconstruction::correct3D_RS(out, psf, out, 1.0 / SNR, n_threads); } - Log::print("Writing output"); + if (!do_multiple) Log::print("Writing output"); const double samplingRate = tomogramSet.getPixelSize(tomoIndex) * spacing; out.write(getOutputFileName(), samplingRate); @@ -299,12 +303,9 @@ void TomoBackprojectProgram::reconstructOneTomogram(int tomoIndex) FileName TomoBackprojectProgram::getOutputFileName(int index) { // If we're reconstructing many tomograms, or the output filename is a directory: use standardized output filenames - if (tomoName == "*" || outFn[outFn.size()-1] == '/') + if (do_multiple) { - FileName result = outFn; - if (result[result.size()-1] != '/') result += "/"; - result += "tomograms/rec_" + tomogramSet.getTomogramName(index)+".mrc"; - return result; + return outFn + "tomograms/rec_" + tomogramSet.getTomogramName(index)+".mrc"; } else { diff --git a/src/jaz/tomography/programs/reconstruct_tomogram.h b/src/jaz/tomography/programs/reconstruct_tomogram.h index 2cea6a0dd..2118ae665 100644 --- a/src/jaz/tomography/programs/reconstruct_tomogram.h +++ b/src/jaz/tomography/programs/reconstruct_tomogram.h @@ -21,7 +21,7 @@ class TomoBackprojectProgram double spacing, x0, y0, z0, taperDist, taperFalloff; FileName tomoName, outFn; bool applyPreWeight, applyWeight, applyCtf, zeroDC, FourierCrop; - bool do_only_unfinished; + bool do_multiple, do_only_unfinished; double SNR; std::vector tomoIndexTodo; diff --git a/src/jaz/tomography/tomogram_set.cpp b/src/jaz/tomography/tomogram_set.cpp index cee3c0673..a9ed97309 100644 --- a/src/jaz/tomography/tomogram_set.cpp +++ b/src/jaz/tomography/tomogram_set.cpp @@ -160,9 +160,18 @@ Tomogram TomogramSet::loadTomogram(int index, bool loadImageData) const globalTable.getValueSafely(EMDL_TOMO_NAME, tomoName, index); const MetaDataTable& m = tomogramTables[index]; - globalTable.getValueSafely(EMDL_TOMO_SIZE_X, out.w0, index); - globalTable.getValueSafely(EMDL_TOMO_SIZE_Y, out.h0, index); - globalTable.getValueSafely(EMDL_TOMO_SIZE_Z, out.d0, index); + if (globalTable.containsLabel(EMDL_TOMO_SIZE_X) && + globalTable.containsLabel(EMDL_TOMO_SIZE_Y) && + globalTable.containsLabel(EMDL_TOMO_SIZE_Z)) + { + globalTable.getValueSafely(EMDL_TOMO_SIZE_X, out.w0, index); + globalTable.getValueSafely(EMDL_TOMO_SIZE_Y, out.h0, index); + globalTable.getValueSafely(EMDL_TOMO_SIZE_Z, out.d0, index); + } + else + { + out.w0 = out.h0 = out.d0 = -999; + } if (globalTable.containsLabel(EMDL_TOMO_TILT_SERIES_NAME)) { @@ -204,7 +213,7 @@ Tomogram TomogramSet::loadTomogram(int index, bool loadImageData) const { REPORT_ERROR("ERROR: tomogramTable for " + tomoName + " does not contain a rlnMicrographName label, yet the globalTable also does not contain a rlnTomoTiltSeriesName label!"); } - + std::string fn_img; m.getValueSafely(EMDL_MICROGRAPH_NAME, fn_img, 0); Image I; diff --git a/src/metadata_label.h b/src/metadata_label.h index 5c5a77a22..031b91dd0 100644 --- a/src/metadata_label.h +++ b/src/metadata_label.h @@ -605,6 +605,7 @@ enum EMDLabel EMDL_TOMO_NAME, EMDL_TOMO_TILT_SERIES_NAME, EMDL_TOMO_TILT_SERIES_STARFILE, + EMDL_TOMO_TILT_MOVIE_FRAMECOUNT, EMDL_TOMO_FRAME_COUNT, EMDL_TOMO_RECONSTRUCTED_TOMOGRAM_FILE_NAME, EMDL_TOMO_SIZE_X, @@ -627,6 +628,9 @@ enum EMDLabel EMDL_TOMO_MANIFOLD_TYPE, EMDL_TOMO_MANIFOLD_PARAMETERS, EMDL_TOMO_DEFOCUS_SLOPE, + EMDL_TOMO_NOMINAL_TILT_STAGE_ANGLE, + EMDL_TOMO_NOMINAL_TILT_AXIS_ANGLE, + EMDL_TOMO_NOMINAL_DEFOCUS, EMDL_TOMO_PARTICLES_FILE_NAME, EMDL_TOMO_TOMOGRAMS_FILE_NAME, @@ -1289,6 +1293,7 @@ class StaticInitialization EMDL::addLabel(EMDL_TOMO_NAME, EMDL_STRING, "rlnTomoName", "Arbitrary name for a tomogram"); EMDL::addLabel(EMDL_TOMO_TILT_SERIES_NAME, EMDL_STRING, "rlnTomoTiltSeriesName", "Tilt series file name"); EMDL::addLabel(EMDL_TOMO_TILT_SERIES_STARFILE, EMDL_STRING, "rlnTomoTiltSeriesStarFile", "Tilt series starfile"); + EMDL::addLabel(EMDL_TOMO_TILT_MOVIE_FRAMECOUNT, EMDL_INT, "rlnTomoTiltMovieFrameCount", "Number of frames in the tilt series movies"); EMDL::addLabel(EMDL_TOMO_FRAME_COUNT, EMDL_INT, "rlnTomoFrameCount", "Number of tilts in a tilt series"); EMDL::addLabel(EMDL_TOMO_RECONSTRUCTED_TOMOGRAM_FILE_NAME, EMDL_STRING, "rlnTomoReconstructedTomogram", "File name of a reconstructed tomogram"); EMDL::addLabel(EMDL_TOMO_FRAME_COUNT, EMDL_INT, "rlnTomoFrameCount", "Number of tilts in a tilt series"); @@ -1325,6 +1330,10 @@ class StaticInitialization EMDL::addLabel(EMDL_TOMO_REFERENCE_MAP_2_FILE_NAME, EMDL_STRING, "rlnTomoReferenceMap2File", "Name of second reference map file"); EMDL::addLabel(EMDL_TOMO_REFERENCE_MASK_FILE_NAME, EMDL_STRING, "rlnTomoReferenceMaskFile", "Name of mask file corresponding to a pair of reference maps"); EMDL::addLabel(EMDL_TOMO_REFERENCE_FSC_FILE_NAME, EMDL_STRING, "rlnTomoReferenceFscFile", "Name of FSC STAR file corresponding to a pair of reference maps"); + EMDL::addLabel(EMDL_TOMO_NOMINAL_TILT_STAGE_ANGLE, EMDL_DOUBLE, "rlnTomoNominalStageTiltAngle", "Nominal value for the stage tilt angle"); + EMDL::addLabel(EMDL_TOMO_NOMINAL_TILT_AXIS_ANGLE, EMDL_DOUBLE, "rlnTomoNominalTiltAxisAngle", "Nominal value for the angle of the tilt axis"); + EMDL::addLabel(EMDL_TOMO_NOMINAL_DEFOCUS, EMDL_DOUBLE, "rlnTomoNominalDefocus", "Nominal value for the defocus in the tilt series image"); + EMDL::addLabel(EMDL_TOMO_IMPORT_OFFSET_X, EMDL_DOUBLE, "rlnTomoImportOffsetX", "X offset of a tomogram"); EMDL::addLabel(EMDL_TOMO_IMPORT_OFFSET_Y, EMDL_DOUBLE, "rlnTomoImportOffsetY", "Y offset of a tomogram"); From f1d6781011a6ad6011a1a2f04c2c572135e60ba7 Mon Sep 17 00:00:00 2001 From: scheres Date: Wed, 27 Apr 2022 15:20:48 +0100 Subject: [PATCH 022/495] forgot to apply spacing to dimensions --- src/jaz/tomography/programs/reconstruct_tomogram.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/jaz/tomography/programs/reconstruct_tomogram.cpp b/src/jaz/tomography/programs/reconstruct_tomogram.cpp index 1ce5f670b..49f2eb28e 100644 --- a/src/jaz/tomography/programs/reconstruct_tomogram.cpp +++ b/src/jaz/tomography/programs/reconstruct_tomogram.cpp @@ -166,6 +166,10 @@ void TomoBackprojectProgram::reconstructOneTomogram(int tomoIndex) std::vector projAct(fc); double pixelSizeAct = tomogram.optics.pixelSize; + const int w1 = w / spacing + 0.5; + const int h1 = h / spacing + 0.5; + const int t1 = d / spacing; + if (std::abs(spacing - 1.0) < 1e-2) { projAct = tomogram.projectionMatrices; @@ -208,7 +212,7 @@ void TomoBackprojectProgram::reconstructOneTomogram(int tomoIndex) d3Vector orig(x0, y0, z0); - BufferedImage out(w, h, d); + BufferedImage out(w1, h1, t1); out.fill(0.f); BufferedImage psfStack; @@ -272,7 +276,7 @@ void TomoBackprojectProgram::reconstructOneTomogram(int tomoIndex) if (applyWeight || applyCtf) { - BufferedImage psf(w, h, d); + BufferedImage psf(w1, h1, t1); psf.fill(0.f); if (applyCtf) From 5cc1b1eb0433b4a95ecd5e9dc4e6347c0764d5ca Mon Sep 17 00:00:00 2001 From: scheres Date: Wed, 27 Apr 2022 15:32:43 +0100 Subject: [PATCH 023/495] debugged getoutputfilename --- src/jaz/tomography/programs/reconstruct_tomogram.cpp | 9 ++++++--- src/jaz/tomography/programs/reconstruct_tomogram.h | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/jaz/tomography/programs/reconstruct_tomogram.cpp b/src/jaz/tomography/programs/reconstruct_tomogram.cpp index 49f2eb28e..846d6f529 100644 --- a/src/jaz/tomography/programs/reconstruct_tomogram.cpp +++ b/src/jaz/tomography/programs/reconstruct_tomogram.cpp @@ -297,10 +297,13 @@ void TomoBackprojectProgram::reconstructOneTomogram(int tomoIndex) if (!do_multiple) Log::print("Writing output"); const double samplingRate = tomogramSet.getPixelSize(tomoIndex) * spacing; - out.write(getOutputFileName(), samplingRate); + out.write(getOutputFileName(tomoIndex), samplingRate); - // Also add the tomogram name to the tomogramSet - tomogramSet.globalTable.setValue(EMDL_TOMO_RECONSTRUCTED_TOMOGRAM_FILE_NAME, getOutputFileName(), tomoIndex); + // Also add the tomogram sizes and name to the tomogramSet + tomogramSet.globalTable.setValue(EMDL_TOMO_SIZE_X, w, tomoIndex); + tomogramSet.globalTable.setValue(EMDL_TOMO_SIZE_Y, h, tomoIndex); + tomogramSet.globalTable.setValue(EMDL_TOMO_SIZE_Z, d, tomoIndex); + tomogramSet.globalTable.setValue(EMDL_TOMO_RECONSTRUCTED_TOMOGRAM_FILE_NAME, getOutputFileName(tomoIndex), tomoIndex); } diff --git a/src/jaz/tomography/programs/reconstruct_tomogram.h b/src/jaz/tomography/programs/reconstruct_tomogram.h index 2118ae665..22449d0a5 100644 --- a/src/jaz/tomography/programs/reconstruct_tomogram.h +++ b/src/jaz/tomography/programs/reconstruct_tomogram.h @@ -34,7 +34,7 @@ class TomoBackprojectProgram void reconstructOneTomogram(int tomoIndex); private: - FileName getOutputFileName(int index = -1); + FileName getOutputFileName(int index); }; #endif From 8bbf740d543ce04225336ff617f66e842fa755e0 Mon Sep 17 00:00:00 2001 From: scheres Date: Wed, 27 Apr 2022 17:26:05 +0100 Subject: [PATCH 024/495] remove spurious output directory if no slash given at end of output name --- src/jaz/tomography/programs/reconstruct_tomogram.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/jaz/tomography/programs/reconstruct_tomogram.cpp b/src/jaz/tomography/programs/reconstruct_tomogram.cpp index 846d6f529..35890b3aa 100644 --- a/src/jaz/tomography/programs/reconstruct_tomogram.cpp +++ b/src/jaz/tomography/programs/reconstruct_tomogram.cpp @@ -80,10 +80,6 @@ void TomoBackprojectProgram::initialise() if (!tomogramSet.read(optimisationSet.tomograms)) REPORT_ERROR("ERROR: there was a problem reading the tomogram set"); - // Make sure output directory exists (this will work for one or all tomograms) - FileName fn_tmp = getOutputFileName(0); - if (!exists(fn_tmp.beforeLastOf("/"))) mktree(fn_tmp.beforeLastOf("/")); - tomoIndexTodo.clear(); if (tomoName == "*") From 1e5623b8a84f378269261f515cf56e98786dc867 Mon Sep 17 00:00:00 2001 From: scheres Date: Thu, 28 Apr 2022 09:21:44 +0100 Subject: [PATCH 025/495] mktree just before getOutputFilename for unexpected tomogram names with subdirectories --- .../programs/reconstruct_tomogram.cpp | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/jaz/tomography/programs/reconstruct_tomogram.cpp b/src/jaz/tomography/programs/reconstruct_tomogram.cpp index 35890b3aa..a66dedbda 100644 --- a/src/jaz/tomography/programs/reconstruct_tomogram.cpp +++ b/src/jaz/tomography/programs/reconstruct_tomogram.cpp @@ -95,15 +95,12 @@ void TomoBackprojectProgram::initialise() if (outFn[outFn.size()-1] != '/') outFn += '/'; FileName fn_dir = outFn + "tomograms/"; - if (!exists(fn_dir)) mktree(fn_dir); } else { tomoIndexTodo.push_back(tomogramSet.getTomogramIndex(tomoName)); do_multiple = false; - - if (!exists(outFn.beforeLastOf("/"))) mktree(outFn.beforeLastOf("/")); } std::cout << " + Reconstructing " << tomoIndexTodo.size() << " tomograms: " << std::endl; @@ -306,14 +303,12 @@ void TomoBackprojectProgram::reconstructOneTomogram(int tomoIndex) FileName TomoBackprojectProgram::getOutputFileName(int index) { // If we're reconstructing many tomograms, or the output filename is a directory: use standardized output filenames - if (do_multiple) - { - return outFn + "tomograms/rec_" + tomogramSet.getTomogramName(index)+".mrc"; - } - else - { - return outFn; - } + FileName fn_result = outFn; + + if (do_multiple) fn_result += "tomograms/rec_" + tomogramSet.getTomogramName(index)+".mrc"; + + if (!exists(fn_result.beforeLastOf("/"))) mktree(fn_result.beforeLastOf("/")); + return fn_result; } \ No newline at end of file From c1e8ea6d56c319c8cf67e64d86761b1b41d2680b Mon Sep 17 00:00:00 2001 From: scheres Date: Fri, 6 May 2022 15:02:11 +0100 Subject: [PATCH 026/495] added --rescale_coords option to relion_tomo_sutomo on request of Alister Burt --- src/jaz/tomography/programs/subtomo.cpp | 26 ++++++++++++++++++++++++- src/jaz/tomography/programs/subtomo.h | 5 +++-- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/src/jaz/tomography/programs/subtomo.cpp b/src/jaz/tomography/programs/subtomo.cpp index b42884c9a..64ec59246 100644 --- a/src/jaz/tomography/programs/subtomo.cpp +++ b/src/jaz/tomography/programs/subtomo.cpp @@ -47,6 +47,7 @@ void SubtomoProgram::readBasicParameters(IOParser& parser) boxSize = textToInteger(parser.getOption("--b", "Binned projection box size")); cropSize = textToInteger(parser.getOption("--crop", "Output box size", "-1")); binning = textToDouble(parser.getOption("--bin", "Binning factor", "1")); + rescale_coords = textToDouble(parser.getOption("--rescale_coords", "Rescale input particles by this factor", "1.")); write_multiplicity = parser.checkOption("--multi", "Write out multiplicity volumes"); SNR = textToDouble(parser.getOption("--SNR", "Assumed signal-to-noise ratio (negative means use a heuristic)", "-1")); @@ -212,7 +213,7 @@ void SubtomoProgram::run() } void SubtomoProgram::initialise( - const ParticleSet& particleSet, + ParticleSet& particleSet, const std::vector>& particles, const TomogramSet& tomogramSet) { @@ -251,6 +252,29 @@ void SubtomoProgram::initialise( } } + if (fabs(1. - rescale_coords) > 1e-3) + { + Log::print("Rescaling input coordinates ... "); + + for (int t = 0; t < tc; t++) + { + const int pc = particles[t].size(); + + if (pc == 0) continue; + + for (int p = 0; p < pc; p++) + { + const ParticleIndex part_id = particles[t][p]; + + d3Vector pos = particleSet.getParticleCoord(part_id); + + pos *= rescale_coords; + + particleSet.setParticleCoord(part_id, pos); + } + } + } + writeParticleSet(particleSet, particles, tomogramSet); } diff --git a/src/jaz/tomography/programs/subtomo.h b/src/jaz/tomography/programs/subtomo.h index 3df799ac0..5199540fa 100644 --- a/src/jaz/tomography/programs/subtomo.h +++ b/src/jaz/tomography/programs/subtomo.h @@ -32,7 +32,8 @@ class SubtomoProgram double SNR, - binning, + binning, + rescale_coords, taper, env_sigma, cone_slope, @@ -72,7 +73,7 @@ class SubtomoProgram protected: void initialise( - const ParticleSet& particleSet, + ParticleSet& particleSet, const std::vector>& particles, const TomogramSet& tomogramSet); From dcd02aa5a7c35e125f8a43ac4b737742126105a1 Mon Sep 17 00:00:00 2001 From: scheres Date: Fri, 6 May 2022 15:32:34 +0100 Subject: [PATCH 027/495] write out tomogram binning factor in tomograms.star --- src/jaz/tomography/programs/reconstruct_tomogram.cpp | 2 ++ src/jaz/tomography/programs/subtomo.cpp | 4 ++-- src/metadata_label.h | 6 ++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/jaz/tomography/programs/reconstruct_tomogram.cpp b/src/jaz/tomography/programs/reconstruct_tomogram.cpp index a66dedbda..6c4a90a1c 100644 --- a/src/jaz/tomography/programs/reconstruct_tomogram.cpp +++ b/src/jaz/tomography/programs/reconstruct_tomogram.cpp @@ -192,6 +192,8 @@ void TomoBackprojectProgram::reconstructOneTomogram(int tomoIndex) } pixelSizeAct *= spacing; + + tomogramSet.globalTable.setValue(EMDL_TOMO_TOMOGRAM_BINNING, spacing, tomoIndex); } else { diff --git a/src/jaz/tomography/programs/subtomo.cpp b/src/jaz/tomography/programs/subtomo.cpp index 64ec59246..d8647d11d 100644 --- a/src/jaz/tomography/programs/subtomo.cpp +++ b/src/jaz/tomography/programs/subtomo.cpp @@ -252,14 +252,14 @@ void SubtomoProgram::initialise( } } - if (fabs(1. - rescale_coords) > 1e-3) + if (std::abs(rescale_coords - 1.0) > 1e-2) { Log::print("Rescaling input coordinates ... "); for (int t = 0; t < tc; t++) { const int pc = particles[t].size(); - + if (pc == 0) continue; for (int p = 0; p < pc; p++) diff --git a/src/metadata_label.h b/src/metadata_label.h index 031b91dd0..9b96240ff 100644 --- a/src/metadata_label.h +++ b/src/metadata_label.h @@ -622,7 +622,8 @@ enum EMDLabel EMDL_TOMO_SUBTOMOGRAM_TILT, EMDL_TOMO_SUBTOMOGRAM_PSI, EMDL_TOMO_SUBTOMOGRAM_BINNING, - EMDL_TOMO_PARTICLE_NAME, + EMDL_TOMO_TOMOGRAM_BINNING, + EMDL_TOMO_PARTICLE_NAME, EMDL_TOMO_PARTICLE_ID, EMDL_TOMO_MANIFOLD_INDEX, EMDL_TOMO_MANIFOLD_TYPE, @@ -1314,7 +1315,8 @@ class StaticInitialization EMDL::addLabel(EMDL_TOMO_SUBTOMOGRAM_TILT, EMDL_DOUBLE, "rlnTomoSubtomogramTilt", "Second Euler angle of a subtomogram (tilt, in degrees)"); EMDL::addLabel(EMDL_TOMO_SUBTOMOGRAM_PSI, EMDL_DOUBLE, "rlnTomoSubtomogramPsi", "Third Euler angle of a subtomogram (psi, in degrees)"); EMDL::addLabel(EMDL_TOMO_SUBTOMOGRAM_BINNING, EMDL_DOUBLE, "rlnTomoSubtomogramBinning", "Binning level of a subtomogram"); - EMDL::addLabel(EMDL_TOMO_PARTICLE_NAME, EMDL_STRING, "rlnTomoParticleName", "Name of each individual particle"); + EMDL::addLabel(EMDL_TOMO_TOMOGRAM_BINNING, EMDL_DOUBLE, "rlnTomoTomogramBinning", "Binning level of a reconstructed tomogram"); + EMDL::addLabel(EMDL_TOMO_PARTICLE_NAME, EMDL_STRING, "rlnTomoParticleName", "Name of each individual particle"); EMDL::addLabel(EMDL_TOMO_PARTICLE_ID, EMDL_INT, "rlnTomoParticleId", "Unique particle index"); EMDL::addLabel(EMDL_TOMO_MANIFOLD_INDEX, EMDL_INT, "rlnTomoManifoldIndex", "Index of a 2D manifold in a tomogram"); From 4f9cc486919be1a1e5fd740bc9e9c266eb720f48 Mon Sep 17 00:00:00 2001 From: scheres Date: Tue, 31 May 2022 17:45:40 +0100 Subject: [PATCH 028/495] added aretomo support for Alisters IMOD/AreTomo wrapper --- src/align_tiltseries_runner.cpp | 18 +++++++++++++++-- src/align_tiltseries_runner.h | 17 +++++++++++++---- src/gui_jobwindow.cpp | 34 ++++++++++++++++++++++++++------- src/pipeline_jobs.cpp | 24 ++++++++++++++++++++--- src/pipeline_jobs.h | 1 + 5 files changed, 78 insertions(+), 16 deletions(-) diff --git a/src/align_tiltseries_runner.cpp b/src/align_tiltseries_runner.cpp index 60ca98c80..1b7689365 100644 --- a/src/align_tiltseries_runner.cpp +++ b/src/align_tiltseries_runner.cpp @@ -27,6 +27,7 @@ void AlignTiltseriesRunner::read(int argc, char **argv, int rank) fn_out = parser.getOption("--o", "Directory, where all output files will be stored", "AlignTiltSeries/"); continue_old = parser.checkOption("--only_do_unfinished", "Only estimate CTFs for those tomograms for which there is not yet a logfile with Final values."); do_at_most = textToInteger(parser.getOption("--do_at_most", "Only process up to this number of (unprocessed) tomograms.", "-1")); + fn_imodwrapper_exe = parser.getOption("--wrapper_executable", "Alister Burt's wrapper script to call IMOD/AreTomo (default is set through $RELION_IMOD_WRAPPER_EXECUTABLE)", ""); int fid_section = parser.addSection("IMOD fiducial-based alignment options"); do_imod_fiducials = parser.checkOption("--imod_fiducials", "Use IMOD's fiducial-based alignment method"); @@ -37,6 +38,14 @@ void AlignTiltseriesRunner::read(int argc, char **argv, int rank) patch_overlap = textToFloat(parser.getOption("--patch_overlap", "Overlap between the patches (in %)", "10")); patch_size = textToInteger(parser.getOption("--patch_size", "Patch size (in unbinned pixels)", "10")); + int aretomo_section = parser.addSection("AreTomo alignment options"); + do_aretomo = parser.checkOption("--aretomo", "OR: Use AreTomo's alignment method"); + aretomo_resolution = textToFloat(parser.getOption("--aretomo_resolution", "Maximum resolution (in A) to use in AreTomo alignments", "10")); + aretomo_thickness = textToFloat(parser.getOption("--aretomo_thickness", "Thickness (in A) for AreTomo alignment", "2000")); + + patch_overlap = textToFloat(parser.getOption("--patch_overlap", "Overlap between the patches (in %)", "10")); + patch_size = textToInteger(parser.getOption("--patch_size", "Patch size (in unbinned pixels)", "10")); + int exp_section = parser.addSection("Expert options"); other_wrapper_args = parser.checkOption("--other_wrapper_args", "Additional command-line arguments that will be passed onto the wrapper."); @@ -177,7 +186,7 @@ void AlignTiltseriesRunner::run() void AlignTiltseriesRunner::executeImodWrapper(long idx_tomo) { - std::string command = "relion_tomo_align_tilt_series "; + std::string command = fn_imodwrapper_exe + " "; // Make sure the methods are the first argument to the program! if (do_imod_fiducials) { @@ -190,7 +199,12 @@ void AlignTiltseriesRunner::executeImodWrapper(long idx_tomo) command += " --unbinned-patch-size-pixels " + integerToString(patch_size); command += " -- patch-overlap-percentage " + floatToString(patch_overlap); } - + else if (do_aretomo) + { + command += " AreTomo"; + command += " --alignment-resolution " + floatToString(aretomo_resolution); + command += " --alignment-thickness " + floatToString(aretomo_thickness); + } command += " --tilt-series-star-file " + fn_in; command += " --tomogram-name " + tomogramSet.getTomogramName(idx_tomo); command += " --output-directory " + fn_out; diff --git a/src/align_tiltseries_runner.h b/src/align_tiltseries_runner.h index 68f4f7381..c38c5d6ec 100644 --- a/src/align_tiltseries_runner.h +++ b/src/align_tiltseries_runner.h @@ -53,23 +53,32 @@ class AlignTiltseriesRunner TomogramSet tomogramSet; // CTFFIND and Gctf executables and shell - FileName fn_imodwrapper_exe, fn_shell; + FileName fn_imodwrapper_exe; // Use IMOD:fiducials bool do_imod_fiducials; - // Use IMOD:patch-tracking - bool do_imod_patchtrack; - // Nominal fiducial diameter (nm) RFLOAT fiducial_diam; + // Use IMOD:patch-tracking + bool do_imod_patchtrack; + // Unbinned patch size (pixels) int patch_size; // Patch overlap (percentage 0-100) RFLOAT patch_overlap; + // Use AreTomo + bool do_aretomo; + + // Resolution used in AreTomo alignment + RFLOAT aretomo_resolution; + + // Alignment thickness in AreTomo + RFLOAT aretomo_thickness; + // Additional gctf command line options std::string other_wrapper_args; diff --git a/src/gui_jobwindow.cpp b/src/gui_jobwindow.cpp index 804032e54..aaf42c3ed 100644 --- a/src/gui_jobwindow.cpp +++ b/src/gui_jobwindow.cpp @@ -2471,7 +2471,7 @@ void JobWindow::initialiseTomoImportWindow() void JobWindow::initialiseTomoAlignTiltseriesWindow() { - setupTabs(1); + setupTabs(3); tab1->begin(); tab1->label("I/O"); @@ -2479,8 +2479,17 @@ void JobWindow::initialiseTomoAlignTiltseriesWindow() place("in_tiltseries", TOGGLE_DEACTIVATE); - // Add a little spacer - current_y += STEPY/2; + // Add a little spacer + current_y += STEPY/2; + + place("imod_wrapper", TOGGLE_DEACTIVATE); + place("other_wrapper_args", TOGGLE_DEACTIVATE); + + + tab1->end(); + tab2->begin(); + tab2->label("IMOD"); + resetHeight(); group1 = new Fl_Group(WCOL0, MENUHEIGHT, 550, 600-MENUHEIGHT, ""); group1->end(); @@ -2504,11 +2513,22 @@ void JobWindow::initialiseTomoAlignTiltseriesWindow() group2->end(); guientries["do_imod_patchtrack"].cb_menu_i(); // make default active - // Add a little spacer - current_y += STEPY/2; - place("other_wrapper_args", TOGGLE_DEACTIVATE); + tab2->end(); + tab3->begin(); + tab3->label("IMOD"); + resetHeight(); - tab1->end(); + group3 = new Fl_Group(WCOL0, MENUHEIGHT, 550, 600-MENUHEIGHT, ""); + group3->end(); + place("do_aretomo", TOGGLE_DEACTIVATE, group3, false); + group3->begin(); + + place("aretomo_resolution", TOGGLE_DEACTIVATE); + place("aretomo_thickness", TOGGLE_DEACTIVATE); + group2->end(); + guientries["do_aretomo"].cb_menu_i(); // make default active + + tab3->end(); } diff --git a/src/pipeline_jobs.cpp b/src/pipeline_jobs.cpp index ea636a07a..c97e9a772 100644 --- a/src/pipeline_jobs.cpp +++ b/src/pipeline_jobs.cpp @@ -6350,6 +6350,11 @@ void RelionJob::initialiseTomoAlignTiltSeriesJob() joboptions["in_tiltseries"] = JobOption("Input tilt series:", OUTNODE_TOMO_TOMOGRAMS, "", "STAR files (*.star)", "Input tomogram set starfile."); + char *default_location = getenv ("RELION_IMOD_WRAPPER_EXECUTABLE"); + char default_wrapper[] = DEFAULTIMODWRAPPERLOCATION; + if (default_location == NULL) default_location = default_wrapper; + joboptions["imod_wrapper"] = JobOption("Alister Burt's IMOD/AreTomo wrapper:", std::string(default_location), "*", ".", "Location of Alister Burt's IMOD/Wrapper script; or just its executable name if in the path. You can control the default of this field by setting environment variable RELION_IMOD_WRAPPER_EXECUTABLE, or by editing the first few lines in src/gui_jobwindow.h and recompile the code. Note that Alister's script should find the executables to IMOD and AreTomo on its own. See Alister's documentation on how to configure this."); + joboptions["do_imod_fiducials"] = JobOption("Use IMOD:fiducials?", true, "Set to Yes to perform tilt series alignment using fiducials in IMOD."); joboptions["fiducial_diameter"] = JobOption("Fiducial diameter (nm): ", 10, 1, 50, 1, "The diameter of the fiducials (in nm)"); @@ -6358,6 +6363,10 @@ void RelionJob::initialiseTomoAlignTiltSeriesJob() joboptions["patch_size"] = JobOption("Patch size (in unbinned pixels): ", 10, 1, 50, 1, "The size of the patches in unbinned pixels."); joboptions["patch_overlap"] = JobOption("Patch overlap (%): ", 10, 0, 100, 10, "The overlap (0-100%) between the patches."); + joboptions["do_aretomo"] = JobOption("Use AreTomo?", false, "Set to Yes to perform tilt series alignment using UCSF's AreTomo."); + joboptions["aretomo_resolution"] = JobOption("Resolution for AreTomo alignment (in A): ", 10, 1, 50, 1, "The maximum resolution (in A) used for AreTomo alignment. The images will be Fourier cropped to have their Nyquist frequency at this value."); + joboptions["aretomo_thickness"] = JobOption("Thickness for AreTomo alignment (in A): ", 2000, 100, 5000, 100, "The tomogram thickness (in A) used for AreTomo alignment. The images will be Fourier cropped to have their Nyquist frequency at this value."); + joboptions["other_wrapper_args"] = JobOption("Other wrapper arguments:", (std::string)"", "Other arguments that will be passed straight onto the IMOD wrapper."); } @@ -6371,9 +6380,10 @@ bool RelionJob::getCommandsTomoAlignTiltSeriesJob(std::string &outputname, std:: int i = 0; if (joboptions["do_imod_fiducials"].getBoolean()) i++; if (joboptions["do_imod_patchtrack"].getBoolean()) i++; + if (joboptions["do_aretomo"].getBoolean()) i++; if (i != 1) { - error_message = "ERROR: you should (only) select ONE of the alignment methods: IMOD:fiducials or IMOD:patchtracking."; + error_message = "ERROR: you should (only) select ONE of the alignment methods: IMOD:fiducials or IMOD:patchtracking or AreTomo."; return false; } @@ -6394,9 +6404,14 @@ bool RelionJob::getCommandsTomoAlignTiltSeriesJob(std::string &outputname, std:: { command += " --imod_patchtrack"; command += " --patch_size " + joboptions["patch_size"].getString(); - command += " -- patch_overlap " + joboptions["patch_overlap"].getString(); + command += " --patch_overlap " + joboptions["patch_overlap"].getString(); + } + else if (joboptions["do_aretomo"].getBoolean()) + { + command += " --aretomo"; + command += " --aretomo_resolution " + joboptions["aretomo_resolution"].getString(); + command += " --aretomo_thickness " + joboptions["aretomo_thickness"].getString(); } - command += " --i " + joboptions["in_tiltseries"].getString(); Node node(joboptions["in_tiltseries"].getString(), joboptions["in_tiltseries"].node_type); inputNodes.push_back(node); @@ -6405,6 +6420,9 @@ bool RelionJob::getCommandsTomoAlignTiltSeriesJob(std::string &outputname, std:: Node node2(outputname+"aligned_tilt_series.star", LABEL_TOMO_TOMOGRAMS); outputNodes.push_back(node2); + if ((joboptions["imod_wrapper"].getString()).length() > 0) + command += " --wrapper_executable " + joboptions["imod_wrapper"].getString(); + if ((joboptions["other_wrapper_args"].getString()).length() > 0) command += " --other_wrapper_args \" " + joboptions["other_wrapper_args"].getString() + " \""; diff --git a/src/pipeline_jobs.h b/src/pipeline_jobs.h index 8d9d0f3ac..bde4f88fb 100644 --- a/src/pipeline_jobs.h +++ b/src/pipeline_jobs.h @@ -54,6 +54,7 @@ // Our own defaults at LMB are the hard-coded ones #define DEFAULTQSUBLOCATION "/public/EM/RELION/relion/bin/relion_qsub.csh" #define DEFAULTCTFFINDLOCATION "/public/EM/ctffind/ctffind.exe" +#define DEFAULTIMODWRAPPERLOCATION "relion_tomo_align_tilt_series" #define DEFAULTMOTIONCOR2LOCATION "/public/EM/MOTIONCOR2/MotionCor2" #define DEFAULTGCTFLOCATION "/public/EM/Gctf/bin/Gctf" #define DEFAULTTOPAZLOCATION "/public/EM/RELION/topaz" From 7aab5344b007894a9a04f0c502beb4d4237e0fc1 Mon Sep 17 00:00:00 2001 From: scheres Date: Wed, 1 Jun 2022 12:29:00 +0100 Subject: [PATCH 029/495] forgot to add are-tomo to options check --- src/align_tiltseries_runner.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/align_tiltseries_runner.cpp b/src/align_tiltseries_runner.cpp index 1b7689365..80c2282b7 100644 --- a/src/align_tiltseries_runner.cpp +++ b/src/align_tiltseries_runner.cpp @@ -76,7 +76,8 @@ void AlignTiltseriesRunner::initialise(bool is_leader) int i = 0; if (do_imod_fiducials) i++; if (do_imod_patchtrack) i++; - if (i != 1) REPORT_ERROR("ERROR: you need to specify one of these options: --imod_fiducials or --imod_patchtrack"); + if (do_aretomo) i++; + if (i != 1) REPORT_ERROR("ERROR: you need to specify one of these options: --imod_fiducials or --imod_patchtrack or --aretomo"); // Make sure fn_out ends with a slash if (fn_out[fn_out.length()-1] != '/') From cd63f4d20be6adb5e882efdb5cfd23eae22479a9 Mon Sep 17 00:00:00 2001 From: scheres Date: Wed, 1 Jun 2022 12:35:58 +0100 Subject: [PATCH 030/495] added rlnTomoXTilt rlnTomoYTilt rlnTomoZRot rlnTomoXShiftAngst rlnTomoYShiftAngst for Alister Burts tilt series alignmwent setup --- src/metadata_label.h | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/metadata_label.h b/src/metadata_label.h index 9b96240ff..cbcb24a33 100644 --- a/src/metadata_label.h +++ b/src/metadata_label.h @@ -615,6 +615,11 @@ enum EMDLabel EMDL_TOMO_PROJECTION_Y, EMDL_TOMO_PROJECTION_Z, EMDL_TOMO_PROJECTION_W, + EMDL_TOMO_XTILT, + EMDL_TOMO_YTILT, + EMDL_TOMO_ZROT, + EMDL_TOMO_XSHIFT_ANGST, + EMDL_TOMO_YSHIFT_ANGST, EMDL_TOMO_HANDEDNESS, EMDL_TOMO_FIDUCIALS_STARFILE, EMDL_TOMO_TILT_SERIES_PIXEL_SIZE, @@ -1302,6 +1307,12 @@ class StaticInitialization EMDL::addLabel(EMDL_TOMO_SIZE_Y, EMDL_INT, "rlnTomoSizeY", "Height of a bin-1 tomogram in pixels"); EMDL::addLabel(EMDL_TOMO_SIZE_Z, EMDL_INT, "rlnTomoSizeZ", "Depth of a bin-1 tomogram in pixels"); + EMDL::addLabel(EMDL_TOMO_XTILT, EMDL_DOUBLE, "rlnTomoXTilt", "Euler angle for rotation of tomogram around X-axis"); + EMDL::addLabel(EMDL_TOMO_YTILT, EMDL_DOUBLE, "rlnTomoYTilt", "Euler angle for rotation of tomogram around Y-axis"); + EMDL::addLabel(EMDL_TOMO_ZROT, EMDL_DOUBLE, "rlnTomoZRot", "Euler angle for rotation of tomogram around Z-axis"); + EMDL::addLabel(EMDL_TOMO_XSHIFT_ANGST, EMDL_DOUBLE, "rlnTomoXShiftAngst", "X-translation (in A) to align the projection of a tomogram with the tilt series image"); + EMDL::addLabel(EMDL_TOMO_YSHIFT_ANGST, EMDL_DOUBLE, "rlnTomoYShiftAngst", "Y-translation (in A) to align the projection of a tomogram with the tilt series image"); + EMDL::addLabel(EMDL_TOMO_PROJECTION_X, EMDL_DOUBLE_VECTOR, "rlnTomoProjX", "First row of the projection matrix"); EMDL::addLabel(EMDL_TOMO_PROJECTION_Y, EMDL_DOUBLE_VECTOR, "rlnTomoProjY", "Second row of the projection matrix"); EMDL::addLabel(EMDL_TOMO_PROJECTION_Z, EMDL_DOUBLE_VECTOR, "rlnTomoProjZ", "Third row of the projection matrix"); From c96c3398ec61933640ef24084129fbd0e07a01fc Mon Sep 17 00:00:00 2001 From: scheres Date: Wed, 1 Jun 2022 12:54:11 +0100 Subject: [PATCH 031/495] implemented Alsiter Burt's tilt series alignment in tomo_reconstruct_tomogram --- .../programs/reconstruct_tomogram.cpp | 96 +++++++++++++++++++ .../programs/reconstruct_tomogram.h | 1 + src/jaz/tomography/tomogram.h | 2 +- src/jaz/tomography/tomogram_set.cpp | 38 +++++--- src/pipeline_jobs.cpp | 3 + 5 files changed, 124 insertions(+), 16 deletions(-) diff --git a/src/jaz/tomography/programs/reconstruct_tomogram.cpp b/src/jaz/tomography/programs/reconstruct_tomogram.cpp index 6c4a90a1c..e6132c358 100644 --- a/src/jaz/tomography/programs/reconstruct_tomogram.cpp +++ b/src/jaz/tomography/programs/reconstruct_tomogram.cpp @@ -146,6 +146,100 @@ void TomoBackprojectProgram::run(int rank, int size) std::cout << " Done!" << std::endl; } +void TomoBackprojectProgram::getProjectMatrices(Tomogram &tomogram, MetaDataTable &tomogramTable) +{ +/* From Alister Burt + * + * tilt_image_center = tilt_image_dimensions / 2 + * specimen_center = tomogram_dimensions / 2 + * + * # Transformations, defined in order of application + * s0 = S(-specimen_center) # put specimen center-of-rotation at the origin + * r0 = Rx(euler_angles['rlnTomoXTilt']) # rotate specimen around X-axis + * r1 = Ry(euler_angles['rlnTomoYTilt']) # rotate specimen around Y-axis + * r2 = Rz(euler_angles['rlnTomoZRot']) # rotate specimen around Z-axis + * s1 = S(specimen_shifts) # shift projected specimen in xy (camera) plane + * s2 = S(tilt_image_center) # move specimen back into tilt-image coordinate system + * + * # compose matrices + * transformations = s2 @ s1 @ r2 @ r1 @ r0 @ s0 + * + */ + + if (!(tomogramTable.containsLabel(EMDL_TOMO_XTILT) && + tomogramTable.containsLabel(EMDL_TOMO_YTILT) && + tomogramTable.containsLabel(EMDL_TOMO_ZROT) && + tomogramTable.containsLabel(EMDL_TOMO_XSHIFT_ANGST) && + tomogramTable.containsLabel(EMDL_TOMO_YSHIFT_ANGST))) + REPORT_ERROR("ERROR: input tomograms.star file does not contain projections matrices, NOR rlnTomoXTilt, rlnTomoYtilt, rlnTomoZRot, rlnTomoXShiftAngst or rlnTomoYShiftAng"); + + double pixelSizeAct = tomogram.optics.pixelSize; + const int fc = tomogram.frameCount; + for (int f = 0; f < fc; f++) + { + d4Matrix s0, s1, s2, r0, r1, r2, out; + + // Get specimen center + t3Vector specimen_center((double)int(w/2), (double)int(h/2), (double)int(d/2) ); + s0 = s0. translation(-specimen_center); + + // Get specimen shifts (in pixels) + double xshift, yshift; + tomogramTable.getValueSafely(EMDL_TOMO_XSHIFT_ANGST, xshift, f); + tomogramTable.getValueSafely(EMDL_TOMO_YSHIFT_ANGST, yshift, f); + t3Vector specimen_shifts(xshift/pixelSizeAct, yshift/pixelSizeAct, 0.); + s1 = s1.translation(specimen_shifts); + + // Get tilt image center + std::vector tilt_image_center_int = tomogram.stack.getSizeVector(); + t3Vector tilt_image_center((double)int(tilt_image_center_int[0]/2), (double)int(tilt_image_center_int[1]/2), 0.); + s2 = s2.translation(tilt_image_center); + + // Get rotation matrices + t3Vector xaxis(1., 0., 0.), yaxis(0., 1., 0.), zaxis(0., 0., 1.); + double xtilt, ytilt, zrot; + tomogramTable.getValueSafely(EMDL_TOMO_XTILT, xtilt, f); + tomogramTable.getValueSafely(EMDL_TOMO_YTILT, ytilt, f); + tomogramTable.getValueSafely(EMDL_TOMO_ZROT, zrot, f); + + r0 = r0.rotation(xaxis, xtilt); + r1 = r1.rotation(yaxis, ytilt); + r2 = r2.rotation(zaxis, zrot); + + tomogram.projectionMatrices[f] = s2 * s1 * r2 * r1 * r0 * s0; + + /* + std::cerr << " f= " << f << " ==============================" << std::endl; + std::cerr << " xtilt= " << xtilt << " ytilt= " << ytilt << " zrot= " << zrot < projAct(fc); double pixelSizeAct = tomogram.optics.pixelSize; + if (!tomogram.hasMatrices) getProjectMatrices(tomogram, tomogramSet.tomogramTables[tomoIndex]); + const int w1 = w / spacing + 0.5; const int h1 = h / spacing + 0.5; const int t1 = d / spacing; diff --git a/src/jaz/tomography/programs/reconstruct_tomogram.h b/src/jaz/tomography/programs/reconstruct_tomogram.h index 22449d0a5..01f15c7c4 100644 --- a/src/jaz/tomography/programs/reconstruct_tomogram.h +++ b/src/jaz/tomography/programs/reconstruct_tomogram.h @@ -31,6 +31,7 @@ class TomoBackprojectProgram void readParameters(int argc, char *argv[]); void initialise(); void run(int rank = 0, int size = 1); + void getProjectMatrices(Tomogram &tomogram, MetaDataTable &tomogramTable); void reconstructOneTomogram(int tomoIndex); private: diff --git a/src/jaz/tomography/tomogram.h b/src/jaz/tomography/tomogram.h index 4159cfe7c..0535935cf 100644 --- a/src/jaz/tomography/tomogram.h +++ b/src/jaz/tomography/tomogram.h @@ -19,7 +19,7 @@ class Tomogram Tomogram(); - bool hasOptics, hasImage, hasDeformations; + bool hasOptics, hasImage, hasDeformations, hasMatrices; OpticsData optics; gravis::i2Vector imageSize; int frameCount; diff --git a/src/jaz/tomography/tomogram_set.cpp b/src/jaz/tomography/tomogram_set.cpp index a9ed97309..49b071ef8 100644 --- a/src/jaz/tomography/tomogram_set.cpp +++ b/src/jaz/tomography/tomogram_set.cpp @@ -285,27 +285,35 @@ Tomogram TomogramSet::loadTomogram(int index, bool loadImageData) const out.cumulativeDose.resize(out.frameCount); out.centralCTFs.resize(out.frameCount); out.projectionMatrices.resize(out.frameCount); + out.hasMatrices = (m.containsLabel(EMDL_TOMO_PROJECTION_X) && + m.containsLabel(EMDL_TOMO_PROJECTION_Y) && + m.containsLabel(EMDL_TOMO_PROJECTION_Z) && + m.containsLabel(EMDL_TOMO_PROJECTION_W)); for (int f = 0; f < out.frameCount; f++) { d4Matrix& P = out.projectionMatrices[f]; - std::vector rows({ - EMDL_TOMO_PROJECTION_X, - EMDL_TOMO_PROJECTION_Y, - EMDL_TOMO_PROJECTION_Z, - EMDL_TOMO_PROJECTION_W }); - - for (int i = 0; i < 4; i++) - { - std::vector vals; - m.getValueSafely(rows[i], vals, f); + if (out.hasMatrices) + { + // Old way of defining tilt series alignments, designed by Jasenko Zivanov + std::vector rows({ + EMDL_TOMO_PROJECTION_X, + EMDL_TOMO_PROJECTION_Y, + EMDL_TOMO_PROJECTION_Z, + EMDL_TOMO_PROJECTION_W }); + + for (int i = 0; i < 4; i++) + { + std::vector vals; + m.getValueSafely(rows[i], vals, f); - for (int j = 0; j < 4; j++) - { - P(i,j) = vals[j]; - } - } + for (int j = 0; j < 4; j++) + { + P(i,j) = vals[j]; + } + } + } CTF& ctf = out.centralCTFs[f]; diff --git a/src/pipeline_jobs.cpp b/src/pipeline_jobs.cpp index 52f2b9b14..ba44a8352 100644 --- a/src/pipeline_jobs.cpp +++ b/src/pipeline_jobs.cpp @@ -6482,6 +6482,9 @@ bool RelionJob::getCommandsTomoReconstructTomogramsJob(std::string &outputname, command += " --h " + joboptions["ydim"].getString(); command += " --d " + joboptions["zdim"].getString(); + // In new version of tilt series alignments by Alister Burt, the origin is again at normal 0,0,0 position + command += " --x0 0 --y0 0 --z0 0 "; + Node node1(outputname+"tomograms.star", LABEL_TOMO_TOMOGRAMS); outputNodes.push_back(node1); From 7022f804bade13659f08f34da865204013a41db7 Mon Sep 17 00:00:00 2001 From: scheres Date: Wed, 1 Jun 2022 13:12:31 +0100 Subject: [PATCH 032/495] raise error if tomoName does not exist --- src/jaz/tomography/programs/reconstruct_tomogram.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/jaz/tomography/programs/reconstruct_tomogram.cpp b/src/jaz/tomography/programs/reconstruct_tomogram.cpp index e6132c358..cf519961b 100644 --- a/src/jaz/tomography/programs/reconstruct_tomogram.cpp +++ b/src/jaz/tomography/programs/reconstruct_tomogram.cpp @@ -99,7 +99,9 @@ void TomoBackprojectProgram::initialise() } else { - tomoIndexTodo.push_back(tomogramSet.getTomogramIndex(tomoName)); + int myidx = tomogramSet.getTomogramIndex(tomoName); + if (myidx < 0) REPORT_ERROR("ERROR: cannot find specific tomogram name \"" + tomoName + "\" in the input star file"); + tomoIndexTodo.push_back(myidx); do_multiple = false; } From 34f75c64d54498c7b5d2d84d797af308949fdae1 Mon Sep 17 00:00:00 2001 From: scheres Date: Wed, 1 Jun 2022 13:12:53 +0100 Subject: [PATCH 033/495] dont provide --tn when tomogram name GUI entry is empty --- src/pipeline_jobs.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/pipeline_jobs.cpp b/src/pipeline_jobs.cpp index ba44a8352..361dca72f 100644 --- a/src/pipeline_jobs.cpp +++ b/src/pipeline_jobs.cpp @@ -1143,7 +1143,12 @@ bool RelionJob::getCommands(std::string &outputname, std::vector &c { result = getCommandsTomoAlignTiltSeriesJob(outputname, commands, final_command, do_makedir, job_counter, error_message); } - else if (type == PROC_TOMO_RECONSTRUCT) + else if (type == PROC_TOMO_RECONSTRUCT_TOMOGRAM) + { + result = getCommandsTomoReconstructTomogramsJob(outputname, commands, final_command, do_makedir, job_counter, error_message); + } + + else if (type == PROC_TOMO_RECONSTRUCT) { result = getCommandsTomoReconPartJob(outputname, commands, final_command, do_makedir, job_counter, error_message); @@ -6476,7 +6481,8 @@ bool RelionJob::getCommandsTomoReconstructTomogramsJob(std::string &outputname, if (error_message != "") return false; command += " --o " + outputname; - command += " --tn " + joboptions["tomo_name"].getString(); + if (joboptions["tomo_name"].getString().length() > 0) + command += " --tn " + joboptions["tomo_name"].getString(); command += " --w " + joboptions["xdim"].getString(); command += " --h " + joboptions["ydim"].getString(); From f5e6bbfcef93d3861c83603d79b99e92313fbb42 Mon Sep 17 00:00:00 2001 From: scheres Date: Wed, 1 Jun 2022 16:52:42 +0100 Subject: [PATCH 034/495] specify pixel size in the binned tomogram --- src/gui_jobwindow.cpp | 8 +++---- .../programs/reconstruct_tomogram.cpp | 23 +++++++------------ .../programs/reconstruct_tomogram.h | 2 +- src/pipeline_jobs.cpp | 10 ++++---- 4 files changed, 19 insertions(+), 24 deletions(-) diff --git a/src/gui_jobwindow.cpp b/src/gui_jobwindow.cpp index 19e2c8dc5..ac2d749c1 100644 --- a/src/gui_jobwindow.cpp +++ b/src/gui_jobwindow.cpp @@ -2540,16 +2540,16 @@ void JobWindow::initialiseTomoReconstructTomogramsWindow() resetHeight(); - place("binning", TOGGLE_DEACTIVATE); - - current_y += STEPY /2 ; - place("xdim", TOGGLE_DEACTIVATE); place("ydim", TOGGLE_DEACTIVATE); place("zdim", TOGGLE_DEACTIVATE); current_y += STEPY /2 ; + place("binned_angpix", TOGGLE_DEACTIVATE); + + current_y += STEPY /2 ; + place("tomo_name"); tab2->end(); diff --git a/src/jaz/tomography/programs/reconstruct_tomogram.cpp b/src/jaz/tomography/programs/reconstruct_tomogram.cpp index cf519961b..d20aad0c5 100644 --- a/src/jaz/tomography/programs/reconstruct_tomogram.cpp +++ b/src/jaz/tomography/programs/reconstruct_tomogram.cpp @@ -56,7 +56,8 @@ void TomoBackprojectProgram::readParameters(int argc, char *argv[]) y0 = textToDouble(parser.getOption("--y0", "Y origin", "1.0")); z0 = textToDouble(parser.getOption("--z0", "Z origin", "1.0")); - spacing = textToDouble(parser.getOption("--bin", "Binning", "8.0")); + spacing = textToDouble(parser.getOption("--bin", "Binning", "1.0")); + angpix_spacing = textToDouble(parser.getOption("--binned_angpix", "OR: desired pixel size after binning", "-1")); n_threads = textToInteger(parser.getOption("--j", "Number of threads", "1")); @@ -210,25 +211,13 @@ void TomoBackprojectProgram::getProjectMatrices(Tomogram &tomogram, MetaDataTabl tomogram.projectionMatrices[f] = s2 * s1 * r2 * r1 * r0 * s0; - /* - std::cerr << " f= " << f << " ==============================" << std::endl; - std::cerr << " xtilt= " << xtilt << " ytilt= " << ytilt << " zrot= " << zrot < projAct(fc); double pixelSizeAct = tomogram.optics.pixelSize; + if (angpix_spacing > 0.) + { + spacing = angpix_spacing / pixelSizeAct; + } + if (!tomogram.hasMatrices) getProjectMatrices(tomogram, tomogramSet.tomogramTables[tomoIndex]); const int w1 = w / spacing + 0.5; diff --git a/src/jaz/tomography/programs/reconstruct_tomogram.h b/src/jaz/tomography/programs/reconstruct_tomogram.h index 01f15c7c4..b0be853d5 100644 --- a/src/jaz/tomography/programs/reconstruct_tomogram.h +++ b/src/jaz/tomography/programs/reconstruct_tomogram.h @@ -18,7 +18,7 @@ class TomoBackprojectProgram int n_threads; int w, h, d; - double spacing, x0, y0, z0, taperDist, taperFalloff; + double spacing, angpix_spacing, x0, y0, z0, taperDist, taperFalloff; FileName tomoName, outFn; bool applyPreWeight, applyWeight, applyCtf, zeroDC, FourierCrop; bool do_multiple, do_only_unfinished; diff --git a/src/pipeline_jobs.cpp b/src/pipeline_jobs.cpp index da42ca1da..7663a7beb 100644 --- a/src/pipeline_jobs.cpp +++ b/src/pipeline_jobs.cpp @@ -6442,11 +6442,11 @@ void RelionJob::initialiseTomoReconstructTomogramsJob() joboptions["tomo_name"] = JobOption("Reconstruct only this tomogram:", std::string(""), "If not left empty, the program will only reconstruct this particular tomogram"); - joboptions["binning"] = JobOption("Binning factor: ", 8, 1, 32, 1, "The tomogram will be downscaled to this factor. For particle picking, often binning to 4 or 8 gives good enough tomograms. Later subtomogram averaging will not use these binned tomograms."); + joboptions["binned_angpix"] = JobOption("Binned pixel size (A): ", 10., 1, 20, 1, "The tomogram will be downscaled to this pixel size. For particle picking, often binning to pixel sizes of 5-10 A gives good enough tomograms. Note that the downsized tomograms are only used for picking; subsequent subtomogram averaging will not use these."); - joboptions["xdim"] = JobOption("Tomogram width (Xdim): ", -1, -1, 6000, 100, "The tomogram X-dimension in pixels. If a negative value is given, this will be determined from the size of the tilt series images."); - joboptions["ydim"] = JobOption("Tomogram height (Ydim): ", -1, -1, 6000, 100, "The tomogram Y-dimension in pixels. If a negative value is given, this will be determined from the size of the tilt series images."); - joboptions["zdim"] = JobOption("Tomogram thickness (Zdim): ", 2000, -1, 6000, 100, "The tomogram Z-dimension in pixels. If a negative value is given, this will be determined from the size of the tilt series images."); + joboptions["xdim"] = JobOption("Unbinned tomogram width (Xdim): ", -1, -1, 6000, 100, "The tomogram X-dimension in pixels. If a negative value is given, this will be determined from the size of the tilt series images."); + joboptions["ydim"] = JobOption("Unbinned tomogram height (Ydim): ", -1, -1, 6000, 100, "The tomogram Y-dimension in pixels. If a negative value is given, this will be determined from the size of the tilt series images."); + joboptions["zdim"] = JobOption("Unbinned tomogram thickness (Zdim): ", 2000, -1, 6000, 100, "The tomogram Z-dimension in pixels. If a negative value is given, this will be determined from the size of the tilt series images."); } @@ -6476,6 +6476,8 @@ bool RelionJob::getCommandsTomoReconstructTomogramsJob(std::string &outputname, command += " --h " + joboptions["ydim"].getString(); command += " --d " + joboptions["zdim"].getString(); + command += " --binned_angpix " + joboptions["binned_angpix"].getString(); + // In new version of tilt series alignments by Alister Burt, the origin is again at normal 0,0,0 position command += " --x0 0 --y0 0 --z0 0 "; From 437542b567cd52406bb858e0b63bede361742c53 Mon Sep 17 00:00:00 2001 From: scheres Date: Thu, 9 Jun 2022 10:00:48 +0100 Subject: [PATCH 035/495] addressing in first instance issue #31 by Euan Pyle; perhaps still need to specify both x and y for patchsize --- src/align_tiltseries_runner.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/align_tiltseries_runner.cpp b/src/align_tiltseries_runner.cpp index 80c2282b7..e4c8a5259 100644 --- a/src/align_tiltseries_runner.cpp +++ b/src/align_tiltseries_runner.cpp @@ -197,8 +197,8 @@ void AlignTiltseriesRunner::executeImodWrapper(long idx_tomo) else if (do_imod_patchtrack) { command += " IMOD:patch-tracking"; - command += " --unbinned-patch-size-pixels " + integerToString(patch_size); - command += " -- patch-overlap-percentage " + floatToString(patch_overlap); + command += " --unbinned-patch-size-pixels " + integerToString(patch_size) + " " + integerToString(patch_size); + command += " --patch-overlap-percentage " + floatToString(patch_overlap); } else if (do_aretomo) { From 59160e8e0e341fe83902594763aaf805d2eab7b3 Mon Sep 17 00:00:00 2001 From: scheres Date: Thu, 9 Jun 2022 14:50:19 +0100 Subject: [PATCH 036/495] provide patch size in Angstroms instead of pixels --- src/align_tiltseries_runner.cpp | 8 ++++++-- src/pipeline_jobs.cpp | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/align_tiltseries_runner.cpp b/src/align_tiltseries_runner.cpp index e4c8a5259..c48fcff01 100644 --- a/src/align_tiltseries_runner.cpp +++ b/src/align_tiltseries_runner.cpp @@ -36,7 +36,7 @@ void AlignTiltseriesRunner::read(int argc, char **argv, int rank) int pat_section = parser.addSection("IMOD patch-tracking alignment options"); do_imod_patchtrack = parser.checkOption("--imod_patchtrack", "OR: Use IMOD's patrick-tracking alignment method"); patch_overlap = textToFloat(parser.getOption("--patch_overlap", "Overlap between the patches (in %)", "10")); - patch_size = textToInteger(parser.getOption("--patch_size", "Patch size (in unbinned pixels)", "10")); + patch_size = textToInteger(parser.getOption("--patch_size", "Patch size (in A)", "10")); int aretomo_section = parser.addSection("AreTomo alignment options"); do_aretomo = parser.checkOption("--aretomo", "OR: Use AreTomo's alignment method"); @@ -187,6 +187,10 @@ void AlignTiltseriesRunner::run() void AlignTiltseriesRunner::executeImodWrapper(long idx_tomo) { + RFLOAT angpix, patch_size_pix; + tomogramSet.globalTable.getValueSafely(EMDL_MICROGRAPH_ORIGINAL_PIXEL_SIZE, angpix, idx_tomo); + patch_size_pix = patch_size / angpix; + std::string command = fn_imodwrapper_exe + " "; // Make sure the methods are the first argument to the program! if (do_imod_fiducials) @@ -197,7 +201,7 @@ void AlignTiltseriesRunner::executeImodWrapper(long idx_tomo) else if (do_imod_patchtrack) { command += " IMOD:patch-tracking"; - command += " --unbinned-patch-size-pixels " + integerToString(patch_size) + " " + integerToString(patch_size); + command += " --unbinned-patch-size-pixels " + floatToString(patch_size_pix) + " " + floatToString(patch_size_pix); command += " --patch-overlap-percentage " + floatToString(patch_overlap); } else if (do_aretomo) diff --git a/src/pipeline_jobs.cpp b/src/pipeline_jobs.cpp index 349216e30..fed676a8c 100644 --- a/src/pipeline_jobs.cpp +++ b/src/pipeline_jobs.cpp @@ -6360,7 +6360,7 @@ void RelionJob::initialiseTomoAlignTiltSeriesJob() joboptions["do_imod_patchtrack"] = JobOption("Use IMOD:patch-tracking?", false, "Set to Yes to perform tilt series alignment using patch-tracking in IMOD."); // TODO: check defaults with the experts - joboptions["patch_size"] = JobOption("Patch size (in unbinned pixels): ", 10, 1, 50, 1, "The size of the patches in unbinned pixels."); + joboptions["patch_size"] = JobOption("Patch size (in A): ", 10, 1, 50, 1, "The size of the patches in Angstrom."); joboptions["patch_overlap"] = JobOption("Patch overlap (%): ", 10, 0, 100, 10, "The overlap (0-100%) between the patches."); joboptions["do_aretomo"] = JobOption("Use AreTomo?", false, "Set to Yes to perform tilt series alignment using UCSF's AreTomo."); From 5b8d198ec74ebe1885a2264b6a7715049a93aa3f Mon Sep 17 00:00:00 2001 From: scheres Date: Thu, 9 Jun 2022 14:51:16 +0100 Subject: [PATCH 037/495] changed TiltSeriesAlign to AlignTiltSeries --- src/pipeline_jobs.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipeline_jobs.h b/src/pipeline_jobs.h index deda1be13..1260004a0 100644 --- a/src/pipeline_jobs.h +++ b/src/pipeline_jobs.h @@ -685,7 +685,7 @@ static int get_node_type(std::string label) #define PROC_TOMO_ALIGN_DIRNAME "FrameAlignTomo" // Frame alignment and particle polishing for subtomography #define PROC_TOMO_RECONSTRUCT_DIRNAME "ReconstructParticleTomo" // Calculation of particle average from the individual tilt series images #define PROC_EXTERNAL_DIRNAME "External" // For running non-relion programs -#define PROC_TOMO_ALIGN_TILTSERIES_DIRNAME "TiltSeriesAlign" // Tilt series alignment for tomogram reconstruction +#define PROC_TOMO_ALIGN_TILTSERIES_DIRNAME "AlignTiltSeries" // Tilt series alignment for tomogram reconstruction #define PROC_TOMO_RECONSTRUCT_TOMOGRAM_DIRNAME "ReconstructTomograms" // Reconstruction of tomograms for particle picking // All the directory names of the different types of jobs defined inside the pipeline From e22feab5d77632fbc3e014c97663bcd71276330a Mon Sep 17 00:00:00 2001 From: scheres Date: Thu, 9 Jun 2022 16:28:34 +0100 Subject: [PATCH 038/495] motioncorr_runner also stores binned pixel size for tomos; tomogramSet.convertBackFromSingleMetaDataTable no longer needs obsModel --- src/ctffind_runner.cpp | 2 +- src/gui_jobwindow.cpp | 2 +- src/jaz/tomography/tomogram_set.cpp | 2 +- src/jaz/tomography/tomogram_set.h | 2 +- src/motioncorr_runner.cpp | 7 +++++-- 5 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/ctffind_runner.cpp b/src/ctffind_runner.cpp index 8771a7b05..33fe2a658 100644 --- a/src/ctffind_runner.cpp +++ b/src/ctffind_runner.cpp @@ -506,7 +506,7 @@ void CtffindRunner::joinCtffindResults() if (is_tomo) { - tomogramSet.convertBackFromSingleMetaDataTable(MDctf, obsModel); + tomogramSet.convertBackFromSingleMetaDataTable(MDctf); tomogramSet.write(fn_out+"tilt_series_ctf.star"); } else diff --git a/src/gui_jobwindow.cpp b/src/gui_jobwindow.cpp index 8aca800b4..064f827c6 100644 --- a/src/gui_jobwindow.cpp +++ b/src/gui_jobwindow.cpp @@ -2511,7 +2511,7 @@ void JobWindow::initialiseTomoAlignTiltseriesWindow() tab2->end(); tab3->begin(); - tab3->label("IMOD"); + tab3->label("AreTomo"); resetHeight(); group3 = new Fl_Group(WCOL0, MENUHEIGHT, 550, 600-MENUHEIGHT, ""); diff --git a/src/jaz/tomography/tomogram_set.cpp b/src/jaz/tomography/tomogram_set.cpp index 49b071ef8..572455738 100644 --- a/src/jaz/tomography/tomogram_set.cpp +++ b/src/jaz/tomography/tomogram_set.cpp @@ -649,7 +649,7 @@ void TomogramSet::generateSingleMetaDataTable(MetaDataTable &MDout, ObservationM } } -void TomogramSet::convertBackFromSingleMetaDataTable(MetaDataTable &MDin, ObservationModel &obsModel) +void TomogramSet::convertBackFromSingleMetaDataTable(MetaDataTable &MDin) { if (!MDin.containsLabel(EMDL_MICROGRAPH_PRE_EXPOSURE)) { diff --git a/src/jaz/tomography/tomogram_set.h b/src/jaz/tomography/tomogram_set.h index 073268797..02880cabb 100644 --- a/src/jaz/tomography/tomogram_set.h +++ b/src/jaz/tomography/tomogram_set.h @@ -65,7 +65,7 @@ class TomogramSet void generateSingleMetaDataTable(MetaDataTable &MDout, ObservationModel &obsModel); // SHWS 6Apr2022: Convert back from one big metadatatable into separate STAR files for each tilt serie - void convertBackFromSingleMetaDataTable(MetaDataTable &MDin, ObservationModel &obsModel); + void convertBackFromSingleMetaDataTable(MetaDataTable &MDin); }; diff --git a/src/motioncorr_runner.cpp b/src/motioncorr_runner.cpp index 4c57b9a5e..a106b7189 100644 --- a/src/motioncorr_runner.cpp +++ b/src/motioncorr_runner.cpp @@ -936,11 +936,14 @@ void MotioncorrRunner::generateLogFilePDFAndWriteStarFiles() RFLOAT my_angpix; obsModel.opticsMdt.getValue(EMDL_MICROGRAPH_ORIGINAL_PIXEL_SIZE, my_angpix); my_angpix *= bin_factor; - obsModel.opticsMdt.setValue(EMDL_MICROGRAPH_PIXEL_SIZE, my_angpix); + if (is_tomo) + tomogramSet.globalTable.setValue(EMDL_MICROGRAPH_PIXEL_SIZE, my_angpix); + else + obsModel.opticsMdt.setValue(EMDL_MICROGRAPH_PIXEL_SIZE, my_angpix); } if (is_tomo) { - tomogramSet.convertBackFromSingleMetaDataTable(MDavg, obsModel); + tomogramSet.convertBackFromSingleMetaDataTable(MDavg); tomogramSet.write(fn_out+"corrected_tilt_series.star"); } else From 6087ba91bd40c976155d5cf52339f52934999652 Mon Sep 17 00:00:00 2001 From: scheres Date: Thu, 9 Jun 2022 16:44:16 +0100 Subject: [PATCH 039/495] added getOriginalPixelSize to tomogram_set; and change now means that reconstruct_tomogram uses the motioncorr-binned micrograph pixel size to set the pixel size in the output tomogram --- src/jaz/tomography/programs/subtomo.cpp | 2 +- src/jaz/tomography/tomogram_set.cpp | 7 ++++++- src/jaz/tomography/tomogram_set.h | 1 + 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/jaz/tomography/programs/subtomo.cpp b/src/jaz/tomography/programs/subtomo.cpp index d58590db5..091eee081 100644 --- a/src/jaz/tomography/programs/subtomo.cpp +++ b/src/jaz/tomography/programs/subtomo.cpp @@ -176,7 +176,7 @@ void SubtomoProgram::run() if (do_sum_all) { - const double pixel_size = binning * tomogramSet.getPixelSize(0); + const double pixel_size = binning * tomogramSet.getOriginalPixelSize(0); sum_data.write(outDir + "sum_data.mrc", pixel_size); Centering::fftwHalfToHumanFull(sum_weights).write(outDir + "sum_weight.mrc", pixel_size); diff --git a/src/jaz/tomography/tomogram_set.cpp b/src/jaz/tomography/tomogram_set.cpp index 572455738..545c05477 100644 --- a/src/jaz/tomography/tomogram_set.cpp +++ b/src/jaz/tomography/tomogram_set.cpp @@ -603,11 +603,16 @@ int TomogramSet::getMaxFrameCount() const return max_val; } -double TomogramSet::getPixelSize(int index) const +double TomogramSet::getOriginalPixelSize(int index) const { return globalTable.getDouble(EMDL_MICROGRAPH_ORIGINAL_PIXEL_SIZE, index); } +double TomogramSet::getPixelSize(int index) const +{ + return globalTable.getDouble(EMDL_MICROGRAPH_PIXEL_SIZE, index); +} + std::string TomogramSet::getOpticsGroupName(int index) const { if (!globalTable.containsLabel(EMDL_IMAGE_OPTICS_GROUP_NAME)) diff --git a/src/jaz/tomography/tomogram_set.h b/src/jaz/tomography/tomogram_set.h index 02880cabb..8af0b7a64 100644 --- a/src/jaz/tomography/tomogram_set.h +++ b/src/jaz/tomography/tomogram_set.h @@ -58,6 +58,7 @@ class TomogramSet int getTomogramIndexSafely(std::string tomogramName) const; int getFrameCount(int index) const; int getMaxFrameCount() const; + double getOriginalPixelSize(int index) const; double getPixelSize(int index) const; std::string getOpticsGroupName(int index) const; From 4cce80bdf6e3a9a2417cc30c051ef912bf6d5796 Mon Sep 17 00:00:00 2001 From: scheres Date: Thu, 9 Jun 2022 16:48:16 +0100 Subject: [PATCH 040/495] check for etomo directive files at the end of align_tiltseries_runner --- src/align_tiltseries_runner.cpp | 35 +++++++++++++++++++++++++++++---- src/align_tiltseries_runner.h | 3 +++ src/metadata_label.h | 4 +++- 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/src/align_tiltseries_runner.cpp b/src/align_tiltseries_runner.cpp index c48fcff01..0d6ac06db 100644 --- a/src/align_tiltseries_runner.cpp +++ b/src/align_tiltseries_runner.cpp @@ -236,19 +236,46 @@ bool AlignTiltseriesRunner::checkImodWrapperResults(long idx_tomo) } +bool AlignTiltseriesRunner::checkEtomoDirectiveFile(long idx_tomo, FileName &filename) +{ + FileName fn_tomo = tomogramSet.getTomogramName(idx_tomo); + filename = fn_out + "external/" + fn_tomo + "/" + fn_tomo + ".edf"; + return exists(filename); + +} + void AlignTiltseriesRunner::joinImodWrapperResults() { // Check again the STAR file exists and has the right labels + // Also check for the presence of any eTomoDirective files + + MetaDataTable MDout; + bool any_edf = false; + for (long itomo = 0; itomo < tomogramSet.size(); itomo++) { - if (!checkImodWrapperResults(itomo)) + if (checkImodWrapperResults(itomo)) { - if (verb) std::cerr << "WARNING: cannot find tilt series alignment parameters in " << tomogramSet.getTomogramName(itomo) << std::endl; + MDout.addObject(tomogramSet.globalTable.getObject(itomo)); + FileName fn_edf; + if (checkEtomoDirectiveFile(itomo, fn_edf)) + { + any_edf = true; + MDout.setValue(EMDL_TOMO_ETOMO_DIRECTIVE_FILE, fn_edf, itomo); + } + else + { + MDout.setValue(EMDL_TOMO_ETOMO_DIRECTIVE_FILE, "undefined", itomo); + } + } + else if (verb) + { + std::cerr << "WARNING: cannot find tilt series alignment parameters in " << tomogramSet.getTomogramName(itomo) << std::endl; } - } + if (!any_edf) MDout.deactivateLabel(EMDL_TOMO_ETOMO_DIRECTIVE_FILE); - tomogramSet.globalTable.write(fn_out + "aligned_tilt_series.star"); + MDout.write(fn_out + "aligned_tilt_series.star"); if (verb > 0) { diff --git a/src/align_tiltseries_runner.h b/src/align_tiltseries_runner.h index c38c5d6ec..2ad4d8992 100644 --- a/src/align_tiltseries_runner.h +++ b/src/align_tiltseries_runner.h @@ -107,6 +107,9 @@ class AlignTiltseriesRunner // Execute CTFFIND for a single tomogram void executeImodWrapper(long idx_tomo); + // Find the etomo directives file (.edf) + bool checkEtomoDirectiveFile(long idx_tomo, FileName &filename); + // Harvest all IMOD results into the single tomograms set starfile, and write it out void joinImodWrapperResults(); diff --git a/src/metadata_label.h b/src/metadata_label.h index 66f949f1f..145a59eb5 100644 --- a/src/metadata_label.h +++ b/src/metadata_label.h @@ -607,6 +607,7 @@ enum EMDLabel EMDL_TOMO_TILT_SERIES_NAME, EMDL_TOMO_TILT_SERIES_STARFILE, EMDL_TOMO_TILT_MOVIE_FRAMECOUNT, + EMDL_TOMO_ETOMO_DIRECTIVE_FILE, EMDL_TOMO_FRAME_COUNT, EMDL_TOMO_RECONSTRUCTED_TOMOGRAM_FILE_NAME, EMDL_TOMO_SIZE_X, @@ -1302,7 +1303,8 @@ class StaticInitialization EMDL::addLabel(EMDL_TOMO_TILT_SERIES_NAME, EMDL_STRING, "rlnTomoTiltSeriesName", "Tilt series file name"); EMDL::addLabel(EMDL_TOMO_TILT_SERIES_STARFILE, EMDL_STRING, "rlnTomoTiltSeriesStarFile", "Tilt series starfile"); EMDL::addLabel(EMDL_TOMO_TILT_MOVIE_FRAMECOUNT, EMDL_INT, "rlnTomoTiltMovieFrameCount", "Number of frames in the tilt series movies"); - EMDL::addLabel(EMDL_TOMO_FRAME_COUNT, EMDL_INT, "rlnTomoFrameCount", "Number of tilts in a tilt series"); + EMDL::addLabel(EMDL_TOMO_ETOMO_DIRECTIVE_FILE, EMDL_STRING, "rlnEtomoDirectiveFile", "Location of the etomo directive file (.edf) from tilt series alignment"); + EMDL::addLabel(EMDL_TOMO_FRAME_COUNT, EMDL_INT, "rlnTomoFrameCount", "Number of tilts in a tilt series"); EMDL::addLabel(EMDL_TOMO_RECONSTRUCTED_TOMOGRAM_FILE_NAME, EMDL_STRING, "rlnTomoReconstructedTomogram", "File name of a reconstructed tomogram"); EMDL::addLabel(EMDL_TOMO_FRAME_COUNT, EMDL_INT, "rlnTomoFrameCount", "Number of tilts in a tilt series"); EMDL::addLabel(EMDL_TOMO_SIZE_X, EMDL_INT, "rlnTomoSizeX", "Width of a bin-1 tomogram in pixels"); From 352f577bad8a099ee945d4462726d661c9ca2c6b Mon Sep 17 00:00:00 2001 From: scheres Date: Fri, 10 Jun 2022 09:17:38 +0100 Subject: [PATCH 041/495] making relion-4.1 purple-blue; still need new image on start page --- src/gui_entries.h | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/gui_entries.h b/src/gui_entries.h index 87741dadf..af0e87a8c 100644 --- a/src/gui_entries.h +++ b/src/gui_entries.h @@ -127,18 +127,18 @@ extern bool create_scheduler_gui; //version-3.1 #define GUI_BUTTON_DARK_COLOR (fl_rgb_color(200, 110, 200)) //version-3.1 #define GUI_RUNBUTTON_COLOR (fl_rgb_color(170, 0, 170)) // Dont forget GUI runbutton colour in src/displayer.h! -//version-3.2 -#define GUI_BUTTON_COLOR (fl_rgb_color(200,80,110)) -#define GUI_BUTTON_DARK_COLOR (fl_rgb_color(170, 40, 70)) -#define GUI_RUNBUTTON_COLOR (fl_rgb_color(160, 30, 60)) +//version-4.0 +//#define GUI_BUTTON_COLOR (fl_rgb_color(200,80,110)) +//#define GUI_BUTTON_DARK_COLOR (fl_rgb_color(170, 40, 70)) +//#define GUI_RUNBUTTON_COLOR (fl_rgb_color(160, 30, 60)) +#define GUI_BACKGROUND_COLOR (fl_rgb_color(230,230,240)) // slightly blue because of blue buttons in 2.0! +#define GUI_BACKGROUND_COLOR2 (fl_rgb_color(180,180,190)) // slightly blue because of blue buttons in 2.0! +//version-4.1 +#define GUI_BUTTON_COLOR (fl_rgb_color(120, 120, 255)) +#define GUI_BUTTON_DARK_COLOR (fl_rgb_color(80, 80, 255)) +#define GUI_RUNBUTTON_COLOR (fl_rgb_color(100, 60, 255)) #define GUI_BACKGROUND_COLOR (fl_rgb_color(230,230,240)) // slightly blue because of blue buttons in 2.0! #define GUI_BACKGROUND_COLOR2 (fl_rgb_color(180,180,190)) // slightly blue because of blue buttons in 2.0! -// devel-version -//#define GUI_BUTTON_COLOR (fl_rgb_color(255, 150, 150)) -//#define GUI_BUTTON_DARK_COLOR (fl_rgb_color(200, 120, 120)) -//#define GUI_RUNBUTTON_COLOR (fl_rgb_color(170, 0, 0)) -//#define GUI_BACKGROUND_COLOR (fl_rgb_color(255,200,200)) // slightly red -//#define GUI_BACKGROUND_COLOR2 (fl_rgb_color(230,180,180)) // slightly red //possible?#define GUI_BUTTON_COLOR (fl_rgb_color(50, 200, 255)) //devel-version //possible #define GUI_RUNBUTTON_COLOR (fl_rgb_color(205,0,155)) From f8f0e74dbabe5a0dc87f02bdcca3c8926a85c010 Mon Sep 17 00:00:00 2001 From: scheres Date: Fri, 10 Jun 2022 14:14:09 +0100 Subject: [PATCH 042/495] patch_size in float, patch_size_pix in int --- src/align_tiltseries_runner.cpp | 10 +++++----- src/align_tiltseries_runner.h | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/align_tiltseries_runner.cpp b/src/align_tiltseries_runner.cpp index 0d6ac06db..bf94a7449 100644 --- a/src/align_tiltseries_runner.cpp +++ b/src/align_tiltseries_runner.cpp @@ -35,8 +35,8 @@ void AlignTiltseriesRunner::read(int argc, char **argv, int rank) int pat_section = parser.addSection("IMOD patch-tracking alignment options"); do_imod_patchtrack = parser.checkOption("--imod_patchtrack", "OR: Use IMOD's patrick-tracking alignment method"); - patch_overlap = textToFloat(parser.getOption("--patch_overlap", "Overlap between the patches (in %)", "10")); - patch_size = textToInteger(parser.getOption("--patch_size", "Patch size (in A)", "10")); + patch_overlap = textToFloat(parser.getOption("--patch_overlap", "Overlap between the patches (in %)", "10.")); + patch_size = textToFloat(parser.getOption("--patch_size", "Patch size (in A)", "10.")); int aretomo_section = parser.addSection("AreTomo alignment options"); do_aretomo = parser.checkOption("--aretomo", "OR: Use AreTomo's alignment method"); @@ -187,9 +187,9 @@ void AlignTiltseriesRunner::run() void AlignTiltseriesRunner::executeImodWrapper(long idx_tomo) { - RFLOAT angpix, patch_size_pix; + RFLOAT angpix; tomogramSet.globalTable.getValueSafely(EMDL_MICROGRAPH_ORIGINAL_PIXEL_SIZE, angpix, idx_tomo); - patch_size_pix = patch_size / angpix; + int patch_size_pix = ROUND(patch_size / angpix); std::string command = fn_imodwrapper_exe + " "; // Make sure the methods are the first argument to the program! @@ -201,7 +201,7 @@ void AlignTiltseriesRunner::executeImodWrapper(long idx_tomo) else if (do_imod_patchtrack) { command += " IMOD:patch-tracking"; - command += " --unbinned-patch-size-pixels " + floatToString(patch_size_pix) + " " + floatToString(patch_size_pix); + command += " --unbinned-patch-size-pixels " + integerToString(patch_size_pix) + " " + integerToString(patch_size_pix); command += " --patch-overlap-percentage " + floatToString(patch_overlap); } else if (do_aretomo) diff --git a/src/align_tiltseries_runner.h b/src/align_tiltseries_runner.h index 2ad4d8992..0849dbbc6 100644 --- a/src/align_tiltseries_runner.h +++ b/src/align_tiltseries_runner.h @@ -65,7 +65,7 @@ class AlignTiltseriesRunner bool do_imod_patchtrack; // Unbinned patch size (pixels) - int patch_size; + RFLOAT patch_size; // Patch overlap (percentage 0-100) RFLOAT patch_overlap; From 93f580609b449efb224b050b8d5c3404ac803e4b Mon Sep 17 00:00:00 2001 From: scheres Date: Wed, 15 Jun 2022 09:52:39 +0100 Subject: [PATCH 043/495] repaired one of Euans reported bugs, not the other_wrapper_args yet... --- src/align_tiltseries_runner.cpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/align_tiltseries_runner.cpp b/src/align_tiltseries_runner.cpp index bf94a7449..9d8b09177 100644 --- a/src/align_tiltseries_runner.cpp +++ b/src/align_tiltseries_runner.cpp @@ -214,7 +214,8 @@ void AlignTiltseriesRunner::executeImodWrapper(long idx_tomo) command += " --tomogram-name " + tomogramSet.getTomogramName(idx_tomo); command += " --output-directory " + fn_out; - command += other_wrapper_args; + if (other_wrapper_args.length() > 0) + command += " " + other_wrapper_args; if (system(command.c_str())) std::cerr << "WARNING: there was an error in executing: " << command << std::endl; @@ -228,11 +229,11 @@ bool AlignTiltseriesRunner::checkImodWrapperResults(long idx_tomo) MetaDataTable MDtomo; MDtomo.read(fn_star); - return (MDtomo.containsLabel(EMDL_ORIENT_ORIGIN_X_ANGSTROM) && - MDtomo.containsLabel(EMDL_ORIENT_ORIGIN_Y_ANGSTROM) && - MDtomo.containsLabel(EMDL_ORIENT_ROT) && - MDtomo.containsLabel(EMDL_ORIENT_TILT) && - MDtomo.containsLabel(EMDL_ORIENT_PSI) ); + return (MDtomo.containsLabel(EMDL_TOMO_XSHIFT_ANGST) && + MDtomo.containsLabel(EMDL_TOMO_YSHIFT_ANGST) && + MDtomo.containsLabel(EMDL_TOMO_XTILT) && + MDtomo.containsLabel(EMDL_TOMO_YTILT) && + MDtomo.containsLabel(EMDL_TOMO_ZROT) ); } From 165092f1239d2dc402727ad3b713f9a2c427a131 Mon Sep 17 00:00:00 2001 From: scheres Date: Wed, 15 Jun 2022 11:03:58 +0100 Subject: [PATCH 044/495] checkOption should be getOption for other_wrapper_args --- src/align_tiltseries_runner.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/align_tiltseries_runner.cpp b/src/align_tiltseries_runner.cpp index 9d8b09177..35d36976b 100644 --- a/src/align_tiltseries_runner.cpp +++ b/src/align_tiltseries_runner.cpp @@ -47,7 +47,7 @@ void AlignTiltseriesRunner::read(int argc, char **argv, int rank) patch_size = textToInteger(parser.getOption("--patch_size", "Patch size (in unbinned pixels)", "10")); int exp_section = parser.addSection("Expert options"); - other_wrapper_args = parser.checkOption("--other_wrapper_args", "Additional command-line arguments that will be passed onto the wrapper."); + other_wrapper_args = parser.getOption("--other_wrapper_args", "Additional command-line arguments that will be passed onto the wrapper.", ""); // Initialise verb for non-parallel execution verb = 1; From bee59f993fdd87f9117b840612f5a173e1bc09bd Mon Sep 17 00:00:00 2001 From: scheres Date: Thu, 16 Jun 2022 09:09:54 +0100 Subject: [PATCH 045/495] add global tablename to output star file --- src/align_tiltseries_runner.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/align_tiltseries_runner.cpp b/src/align_tiltseries_runner.cpp index 35d36976b..7a160c51a 100644 --- a/src/align_tiltseries_runner.cpp +++ b/src/align_tiltseries_runner.cpp @@ -251,6 +251,7 @@ void AlignTiltseriesRunner::joinImodWrapperResults() // Also check for the presence of any eTomoDirective files MetaDataTable MDout; + MDout.setName("global"); bool any_edf = false; for (long itomo = 0; itomo < tomogramSet.size(); itomo++) From 1ddf6f302f0699404748c8bbf1bffc305742f7e8 Mon Sep 17 00:00:00 2001 From: scheres Date: Thu, 16 Jun 2022 09:14:18 +0100 Subject: [PATCH 046/495] set correct global table for tomograms --- src/motioncorr_runner.cpp | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/motioncorr_runner.cpp b/src/motioncorr_runner.cpp index a106b7189..b42f6693a 100644 --- a/src/motioncorr_runner.cpp +++ b/src/motioncorr_runner.cpp @@ -928,37 +928,37 @@ void MotioncorrRunner::generateLogFilePDFAndWriteStarFiles() } + if (verb > 0) progress_bar(fn_ori_micrographs.size()); + // Write out STAR files at the end // In the opticsMdt, set EMDL_MICROGRAPH_PIXEL_SIZE (i.e. possibly binned pixel size). // Keep EMDL_MICROGRAPH_ORIGINAL_PIXEL_SIZE for MTF correction - FOR_ALL_OBJECTS_IN_METADATA_TABLE(obsModel.opticsMdt) - { - RFLOAT my_angpix; - obsModel.opticsMdt.getValue(EMDL_MICROGRAPH_ORIGINAL_PIXEL_SIZE, my_angpix); - my_angpix *= bin_factor; - if (is_tomo) - tomogramSet.globalTable.setValue(EMDL_MICROGRAPH_PIXEL_SIZE, my_angpix); - else - obsModel.opticsMdt.setValue(EMDL_MICROGRAPH_PIXEL_SIZE, my_angpix); - } if (is_tomo) { + FOR_ALL_OBJECTS_IN_METADATA_TABLE(tomogramSet.globalTable) + { + RFLOAT my_angpix; + tomogramSet.globalTable.getValue(EMDL_MICROGRAPH_ORIGINAL_PIXEL_SIZE, my_angpix); + my_angpix *= bin_factor; + tomogramSet.globalTable.setValue(EMDL_MICROGRAPH_PIXEL_SIZE, my_angpix); + } tomogramSet.convertBackFromSingleMetaDataTable(MDavg); tomogramSet.write(fn_out+"corrected_tilt_series.star"); + if (verb > 0) std::cout << " Written: " << fn_out << "corrected_tilt_series.star" << std::endl; } else { + FOR_ALL_OBJECTS_IN_METADATA_TABLE(obsModel.opticsMdt) + { + RFLOAT my_angpix; + obsModel.opticsMdt.getValue(EMDL_MICROGRAPH_ORIGINAL_PIXEL_SIZE, my_angpix); + my_angpix *= bin_factor; + obsModel.opticsMdt.setValue(EMDL_MICROGRAPH_PIXEL_SIZE, my_angpix); + } obsModel.save(MDavg, fn_out + "corrected_micrographs.star", "micrographs"); + if (verb > 0) std::cout << " Written: " << fn_out << "corrected_micrographs.star" << std::endl; } - if (verb > 0 ) - { - progress_bar(fn_ori_micrographs.size()); - - if (is_tomo) std::cout << " Done! Written: " << fn_out << "corrected_tomograms.star" << std::endl; - else std::cout << " Done! Written: " << fn_out << "corrected_micrographs.star" << std::endl; - } - if (verb > 0) std::cout << " Now generating logfile.pdf ... " << std::endl; // Now generate EPS plot with histograms and combine all EPS into a logfile.pdf From 26cac7372e44ba1da8ba754969781e529d655a19 Mon Sep 17 00:00:00 2001 From: scheres Date: Thu, 16 Jun 2022 09:25:50 +0100 Subject: [PATCH 047/495] changed runbutton in relion_display too --- src/displayer.h | 2 +- src/gui_entries.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/displayer.h b/src/displayer.h index afcfa8023..4d2052786 100644 --- a/src/displayer.h +++ b/src/displayer.h @@ -49,7 +49,7 @@ #define GUI_BACKGROUND_COLOR (fl_rgb_color(240,240,240)) #define GUI_INPUT_COLOR (fl_rgb_color(255,255,230)) -#define GUI_RUNBUTTON_COLOR (fl_rgb_color(160, 30, 60)) +#define GUI_RUNBUTTON_COLOR (fl_rgb_color(100, 60, 255)) diff --git a/src/gui_entries.h b/src/gui_entries.h index af0e87a8c..5de1cb83e 100644 --- a/src/gui_entries.h +++ b/src/gui_entries.h @@ -136,6 +136,7 @@ extern bool create_scheduler_gui; //version-4.1 #define GUI_BUTTON_COLOR (fl_rgb_color(120, 120, 255)) #define GUI_BUTTON_DARK_COLOR (fl_rgb_color(80, 80, 255)) +// Dont forget to change runbutton in src/displayer.h too!! #define GUI_RUNBUTTON_COLOR (fl_rgb_color(100, 60, 255)) #define GUI_BACKGROUND_COLOR (fl_rgb_color(230,230,240)) // slightly blue because of blue buttons in 2.0! #define GUI_BACKGROUND_COLOR2 (fl_rgb_color(180,180,190)) // slightly blue because of blue buttons in 2.0! From c157750dfb4b3d6536679f02a6948bac2499f0c0 Mon Sep 17 00:00:00 2001 From: scheres Date: Thu, 16 Jun 2022 09:46:39 +0100 Subject: [PATCH 048/495] use micrograph_pixel_size and NOT micrograph_original_pixel_size for pixelsize of Tomogramgit add ../src/jaz/tomography/tomogram_set.cppgit add ../src/jaz/tomography/tomogram_set.cpp --- src/jaz/tomography/tomogram_set.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jaz/tomography/tomogram_set.cpp b/src/jaz/tomography/tomogram_set.cpp index 545c05477..d4f59088e 100644 --- a/src/jaz/tomography/tomogram_set.cpp +++ b/src/jaz/tomography/tomogram_set.cpp @@ -254,7 +254,7 @@ Tomogram TomogramSet::loadTomogram(int index, bool loadImageData) const double Q0; - globalTable.getValueSafely(EMDL_MICROGRAPH_ORIGINAL_PIXEL_SIZE, out.optics.pixelSize, index); + globalTable.getValueSafely(EMDL_MICROGRAPH_PIXEL_SIZE, out.optics.pixelSize, index); globalTable.getValueSafely(EMDL_CTF_VOLTAGE, out.optics.voltage, index); globalTable.getValueSafely(EMDL_CTF_CS, out.optics.Cs, index); globalTable.getValueSafely(EMDL_CTF_Q0, Q0, index); From 69f00f840c2338b8216bb83ff7cd1cc97a225232 Mon Sep 17 00:00:00 2001 From: scheres Date: Thu, 16 Jun 2022 10:02:32 +0100 Subject: [PATCH 049/495] undo previous commit. Need to think about this.... --- src/jaz/tomography/tomogram_set.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jaz/tomography/tomogram_set.cpp b/src/jaz/tomography/tomogram_set.cpp index d4f59088e..545c05477 100644 --- a/src/jaz/tomography/tomogram_set.cpp +++ b/src/jaz/tomography/tomogram_set.cpp @@ -254,7 +254,7 @@ Tomogram TomogramSet::loadTomogram(int index, bool loadImageData) const double Q0; - globalTable.getValueSafely(EMDL_MICROGRAPH_PIXEL_SIZE, out.optics.pixelSize, index); + globalTable.getValueSafely(EMDL_MICROGRAPH_ORIGINAL_PIXEL_SIZE, out.optics.pixelSize, index); globalTable.getValueSafely(EMDL_CTF_VOLTAGE, out.optics.voltage, index); globalTable.getValueSafely(EMDL_CTF_CS, out.optics.Cs, index); globalTable.getValueSafely(EMDL_CTF_Q0, Q0, index); From 6ae781361da923f7eef1bba7f3b18a46a93392d4 Mon Sep 17 00:00:00 2001 From: scheres Date: Thu, 16 Jun 2022 10:08:41 +0100 Subject: [PATCH 050/495] calculate binned_angpix from pixelsize, not orginalpixelsize --- src/jaz/tomography/programs/reconstruct_tomogram.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/jaz/tomography/programs/reconstruct_tomogram.cpp b/src/jaz/tomography/programs/reconstruct_tomogram.cpp index d20aad0c5..6194373e6 100644 --- a/src/jaz/tomography/programs/reconstruct_tomogram.cpp +++ b/src/jaz/tomography/programs/reconstruct_tomogram.cpp @@ -241,7 +241,8 @@ void TomoBackprojectProgram::reconstructOneTomogram(int tomoIndex) BufferedImage stackAct; std::vector projAct(fc); - double pixelSizeAct = tomogram.optics.pixelSize; + + double pixelSizeAct = tomogramSet.getPixelSize(tomoIndex); if (angpix_spacing > 0.) { From 55291ba50f6e713c5cb6054b49f5c62062cc0d10 Mon Sep 17 00:00:00 2001 From: scheres Date: Thu, 16 Jun 2022 10:09:33 +0100 Subject: [PATCH 051/495] getOriginalPixelSize only through function call --- src/jaz/tomography/particle_set.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jaz/tomography/particle_set.cpp b/src/jaz/tomography/particle_set.cpp index ecc3f7a50..cbacc7641 100644 --- a/src/jaz/tomography/particle_set.cpp +++ b/src/jaz/tomography/particle_set.cpp @@ -247,7 +247,7 @@ d3Vector ParticleSet::getPosition(ParticleIndex particle_id) const { const int og = getOpticsGroup(particle_id); - const double originalPixelSize = optTable.getDouble(EMDL_TOMO_TILT_SERIES_PIXEL_SIZE, og); + const double originalPixelSize = getOriginalPixelSize(og); const d3Matrix A_subtomogram = getSubtomogramMatrix(particle_id); From e1a01156335ee79151b26f3b2a0b7daeb4e02a68 Mon Sep 17 00:00:00 2001 From: scheres Date: Thu, 16 Jun 2022 11:17:40 +0100 Subject: [PATCH 052/495] keep rlnTomoTiltSeriesPixelSize after motioncorr with tomograms, not rlnMicrographPixelSize --- src/align_tiltseries_runner.cpp | 3 +-- src/ctffind_runner.cpp | 8 +++++--- src/ctffind_runner_mpi.cpp | 3 ++- src/jaz/tomography/programs/subtomo.cpp | 2 +- src/jaz/tomography/tomogram_set.cpp | 9 ++++++++- src/motioncorr_runner.cpp | 4 +++- 6 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/align_tiltseries_runner.cpp b/src/align_tiltseries_runner.cpp index 7a160c51a..05beea2e6 100644 --- a/src/align_tiltseries_runner.cpp +++ b/src/align_tiltseries_runner.cpp @@ -187,8 +187,7 @@ void AlignTiltseriesRunner::run() void AlignTiltseriesRunner::executeImodWrapper(long idx_tomo) { - RFLOAT angpix; - tomogramSet.globalTable.getValueSafely(EMDL_MICROGRAPH_ORIGINAL_PIXEL_SIZE, angpix, idx_tomo); + RFLOAT angpix = tomogramSet.getPixelSize(idx_tomo); int patch_size_pix = ROUND(patch_size / angpix); std::string command = fn_imodwrapper_exe + " "; diff --git a/src/ctffind_runner.cpp b/src/ctffind_runner.cpp index 33fe2a658..454096c10 100644 --- a/src/ctffind_runner.cpp +++ b/src/ctffind_runner.cpp @@ -241,7 +241,8 @@ void CtffindRunner::initialise(bool is_leader) obsModel.opticsMdt.setValue(EMDL_CTF_Q0, AmplitudeConstrast); } } - if (!obsModel.opticsMdt.containsLabel(EMDL_MICROGRAPH_PIXEL_SIZE)) + EMDLabel mylabel = (is_tomo) ? EMDL_TOMO_TILT_SERIES_PIXEL_SIZE : EMDL_MICROGRAPH_PIXEL_SIZE; + if (!obsModel.opticsMdt.containsLabel(mylabel)) { if (angpix < 0.) { @@ -249,7 +250,7 @@ void CtffindRunner::initialise(bool is_leader) } FOR_ALL_OBJECTS_IN_METADATA_TABLE(obsModel.opticsMdt) { - obsModel.opticsMdt.setValue(EMDL_MICROGRAPH_PIXEL_SIZE, angpix); + obsModel.opticsMdt.setValue(mylabel, angpix); } } @@ -418,7 +419,8 @@ void CtffindRunner::run() obsModel.opticsMdt.getValue(EMDL_CTF_CS, Cs, optics_group_micrographs[imic]-1); obsModel.opticsMdt.getValue(EMDL_CTF_VOLTAGE, Voltage, optics_group_micrographs[imic]-1); obsModel.opticsMdt.getValue(EMDL_CTF_Q0, AmplitudeConstrast, optics_group_micrographs[imic]-1); - obsModel.opticsMdt.getValue(EMDL_MICROGRAPH_PIXEL_SIZE, angpix, optics_group_micrographs[imic]-1); + EMDLabel mylabel = (is_tomo) ? EMDL_TOMO_TILT_SERIES_PIXEL_SIZE : EMDL_MICROGRAPH_PIXEL_SIZE; + obsModel.opticsMdt.getValue(mylabel, angpix, optics_group_micrographs[imic]-1); if (do_use_gctf) { diff --git a/src/ctffind_runner_mpi.cpp b/src/ctffind_runner_mpi.cpp index a16ef5d2b..fa493c071 100644 --- a/src/ctffind_runner_mpi.cpp +++ b/src/ctffind_runner_mpi.cpp @@ -68,7 +68,8 @@ void CtffindRunnerMpi::run() obsModel.opticsMdt.getValue(EMDL_CTF_CS, Cs, optics_group_micrographs[imic]-1); obsModel.opticsMdt.getValue(EMDL_CTF_VOLTAGE, Voltage, optics_group_micrographs[imic]-1); obsModel.opticsMdt.getValue(EMDL_CTF_Q0, AmplitudeConstrast, optics_group_micrographs[imic]-1); - obsModel.opticsMdt.getValue(EMDL_MICROGRAPH_PIXEL_SIZE, angpix, optics_group_micrographs[imic]-1); + EMDLabel mylabel = (is_tomo) ? EMDL_TOMO_TILT_SERIES_PIXEL_SIZE : EMDL_MICROGRAPH_PIXEL_SIZE; + obsModel.opticsMdt.getValue(mylabel, angpix, optics_group_micrographs[imic]-1); if (do_use_gctf) { diff --git a/src/jaz/tomography/programs/subtomo.cpp b/src/jaz/tomography/programs/subtomo.cpp index 091eee081..d58590db5 100644 --- a/src/jaz/tomography/programs/subtomo.cpp +++ b/src/jaz/tomography/programs/subtomo.cpp @@ -176,7 +176,7 @@ void SubtomoProgram::run() if (do_sum_all) { - const double pixel_size = binning * tomogramSet.getOriginalPixelSize(0); + const double pixel_size = binning * tomogramSet.getPixelSize(0); sum_data.write(outDir + "sum_data.mrc", pixel_size); Centering::fftwHalfToHumanFull(sum_weights).write(outDir + "sum_weight.mrc", pixel_size); diff --git a/src/jaz/tomography/tomogram_set.cpp b/src/jaz/tomography/tomogram_set.cpp index 545c05477..adb9127a8 100644 --- a/src/jaz/tomography/tomogram_set.cpp +++ b/src/jaz/tomography/tomogram_set.cpp @@ -610,7 +610,7 @@ double TomogramSet::getOriginalPixelSize(int index) const double TomogramSet::getPixelSize(int index) const { - return globalTable.getDouble(EMDL_MICROGRAPH_PIXEL_SIZE, index); + return globalTable.getDouble(EMDL_TOMO_TILT_SERIES_PIXEL_SIZE, index); } std::string TomogramSet::getOpticsGroupName(int index) const @@ -645,6 +645,13 @@ void TomogramSet::generateSingleMetaDataTable(MetaDataTable &MDout, ObservationM obsModel.opticsMdt.setValue(EMDL_CTF_CS, Cs); obsModel.opticsMdt.setValue(EMDL_CTF_Q0, Q0); + if (globalTable.containsLabel(EMDL_TOMO_TILT_SERIES_PIXEL_SIZE)) + { + RFLOAT pixSize; + globalTable.getValueSafely(EMDL_TOMO_TILT_SERIES_PIXEL_SIZE, pixSize, t); + obsModel.opticsMdt.setValue(EMDL_TOMO_TILT_SERIES_PIXEL_SIZE, pixSize); + } + FOR_ALL_OBJECTS_IN_METADATA_TABLE(tomogramTables[t]) { tomogramTables[t].setValue(EMDL_IMAGE_OPTICS_GROUP, t+1); diff --git a/src/motioncorr_runner.cpp b/src/motioncorr_runner.cpp index b42f6693a..d0f7a4906 100644 --- a/src/motioncorr_runner.cpp +++ b/src/motioncorr_runner.cpp @@ -940,8 +940,10 @@ void MotioncorrRunner::generateLogFilePDFAndWriteStarFiles() RFLOAT my_angpix; tomogramSet.globalTable.getValue(EMDL_MICROGRAPH_ORIGINAL_PIXEL_SIZE, my_angpix); my_angpix *= bin_factor; - tomogramSet.globalTable.setValue(EMDL_MICROGRAPH_PIXEL_SIZE, my_angpix); + tomogramSet.globalTable.setValue(EMDL_TOMO_TILT_SERIES_PIXEL_SIZE, my_angpix); } + if (tomogramSet.globalTable.containsLabel(EMDL_MICROGRAPH_PIXEL_SIZE)) + tomogramSet.globalTable.deactivateLabel(EMDL_MICROGRAPH_PIXEL_SIZE); tomogramSet.convertBackFromSingleMetaDataTable(MDavg); tomogramSet.write(fn_out+"corrected_tilt_series.star"); if (verb > 0) std::cout << " Written: " << fn_out << "corrected_tilt_series.star" << std::endl; From 7bc1856381314e6d664158e11debedd62a9aa874 Mon Sep 17 00:00:00 2001 From: scheres Date: Thu, 16 Jun 2022 11:32:36 +0100 Subject: [PATCH 053/495] only input_star_mics for motioncorr job --- src/gui_jobwindow.cpp | 3 +-- src/pipeline_jobs.cpp | 28 +++++++--------------------- 2 files changed, 8 insertions(+), 23 deletions(-) diff --git a/src/gui_jobwindow.cpp b/src/gui_jobwindow.cpp index 064f827c6..5a3337c77 100644 --- a/src/gui_jobwindow.cpp +++ b/src/gui_jobwindow.cpp @@ -531,8 +531,7 @@ void JobWindow::initialiseMotioncorrWindow() tab1->label("I/O"); resetHeight(); - if (is_tomo) place("input_tomograms", TOGGLE_DEACTIVATE); - else place("input_star_mics", TOGGLE_DEACTIVATE); + place("input_star_mics", TOGGLE_DEACTIVATE); // Add a little spacer current_y += STEPY/2; diff --git a/src/pipeline_jobs.cpp b/src/pipeline_jobs.cpp index fed676a8c..c5f59462a 100644 --- a/src/pipeline_jobs.cpp +++ b/src/pipeline_jobs.cpp @@ -1388,7 +1388,7 @@ void RelionJob::initialiseMotioncorrJob() if (is_tomo) { - joboptions["input_tomograms"] = JobOption("Input tilt series set: ", OUTNODE_TOMO_TOMOGRAMS, "", "Tomogram set STAR file (*.star)", "Input tomogram set."); + joboptions["input_star_mics"] = JobOption("Input tilt series set: ", OUTNODE_TOMO_TOMOGRAMS, "", "Tomogram set STAR file (*.star)", "Input tomogram set."); } else { @@ -1455,28 +1455,14 @@ bool RelionJob::getCommandsMotioncorrJob(std::string &outputname, std::vector Date: Thu, 16 Jun 2022 11:57:56 +0100 Subject: [PATCH 054/495] only input_star_mics for ctffind job --- src/gui_jobwindow.cpp | 3 +-- src/pipeline_jobs.cpp | 28 +++++++--------------------- 2 files changed, 8 insertions(+), 23 deletions(-) diff --git a/src/gui_jobwindow.cpp b/src/gui_jobwindow.cpp index 5a3337c77..d777630c8 100644 --- a/src/gui_jobwindow.cpp +++ b/src/gui_jobwindow.cpp @@ -599,8 +599,7 @@ void JobWindow::initialiseCtffindWindow() tab1->label("I/O"); resetHeight(); - if (is_tomo) place("input_tomograms", TOGGLE_DEACTIVATE); - else place("input_star_mics", TOGGLE_DEACTIVATE); + place("input_star_mics", TOGGLE_DEACTIVATE); place("use_noDW", TOGGLE_DEACTIVATE); // Add a little spacer diff --git a/src/pipeline_jobs.cpp b/src/pipeline_jobs.cpp index c5f59462a..8f7e9e7a2 100644 --- a/src/pipeline_jobs.cpp +++ b/src/pipeline_jobs.cpp @@ -1622,7 +1622,7 @@ void RelionJob::initialiseCtffindJob() if (is_tomo) { - joboptions["input_tomograms"] = JobOption("Input tilt series set: ", OUTNODE_TOMO_TOMOGRAMS, "", "Tomograms set STAR file (*.star)", "Input tomograms set. ..."); + joboptions["input_star_mics"] = JobOption("Input tilt series set: ", OUTNODE_TOMO_TOMOGRAMS, "", "Tomograms set STAR file (*.star)", "Input tomograms set. ..."); } else { @@ -1706,28 +1706,14 @@ bool RelionJob::getCommandsCtffindJob(std::string &outputname, std::vector Date: Thu, 23 Jun 2022 19:42:29 +0100 Subject: [PATCH 055/495] change default GUI settings to import tilt-series only --- src/pipeline_jobs.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipeline_jobs.cpp b/src/pipeline_jobs.cpp index 8f7e9e7a2..ab24d14a8 100644 --- a/src/pipeline_jobs.cpp +++ b/src/pipeline_jobs.cpp @@ -6078,7 +6078,7 @@ void RelionJob::initialiseTomoImportJob() joboptions["tilt_axis_angle"] = JobOption("Tilt axis angle (deg):", 90.0, 0.0, 180.0, 1.0 , "Nominal value for the angle of the tilt axis"); joboptions["mtf_file"] = JobOption("MTF file:", (std::string)"","MTF file for the detector"); - joboptions["do_tomo"] = JobOption("Import tomograms?", true, "Set this to Yes for importing tomogram directories from IMOD."); + joboptions["do_tomo"] = JobOption("Import tomograms?", false, "Set this to Yes for importing tomogram directories from IMOD."); joboptions["io_tomos"] = JobOption("Append to tomograms set: ", OUTNODE_TOMO_TOMOGRAMS, "", "Tomogram set STAR file (*.star)", "The imported tomograms will be output into this tomogram set. If any tomograms were already in this tomogram set, then the newly imported ones will be added to those."); joboptions["tomo_star"] = JobOption("STAR file with tomograms description: ", "", "Input file (*.star)", ".", "Provide a STAR file with the basic following information to import tomogsrams: \n\n" " - rlnTomoImportImodDir: path to the IMOD directory.\n" From c4b2320b78f7207614633b64adbb97f66cc1c46f Mon Sep 17 00:00:00 2001 From: alisterburt Date: Thu, 23 Jun 2022 19:47:50 +0100 Subject: [PATCH 056/495] Minor updates to GUI labels and defaults in tomo import --- src/pipeline_jobs.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/pipeline_jobs.cpp b/src/pipeline_jobs.cpp index ab24d14a8..355ba4ef1 100644 --- a/src/pipeline_jobs.cpp +++ b/src/pipeline_jobs.cpp @@ -6071,11 +6071,11 @@ void RelionJob::initialiseTomoImportJob() joboptions["Q0"] = JobOption("Amplitude contrast:", 0.1, 0.05, 1, 0.01, "Fraction of amplitude contrast (default=0.1). Often values around 10% work better than theoretically more accurate lower values. "); joboptions["dose"] = JobOption("Frame dose:", 3.0, 0.0, 10.0, 0.1 , "Electron dose (in e/A^2) per frame (image) in the tilt series."); - joboptions["do_tiltseries"]= JobOption("Import tiltseries?", true, "Set this to Yes for importing tilt series straight from serialEM."); - joboptions["movie_files"] = JobOption("Tilt image movie files:", (std::string)"frames/*mrc","File pattern pointing to the raw movie files for the tilt images"); - joboptions["mdoc_files"] = JobOption("mdoc files:", (std::string)"mdoc/*.mdoc","File pattern pointing to the mdoc files from the data acquisition software"); + joboptions["do_tiltseries"]= JobOption("Import tilt-series?", true, "Set this to Yes for importing tilt movies from SerialEM mdoc format metadata."); + joboptions["movie_files"] = JobOption("Tilt image movie files:", (std::string)"frames/*.mrc","File pattern pointing to the raw movie files for the tilt images"); + joboptions["mdoc_files"] = JobOption("mdoc files:", (std::string)"mdoc/*.mdoc","File pattern pointing to the mdoc files."); joboptions["prefix"] = JobOption("Prefix:", (std::string)"","Prefix for XXX"); - joboptions["tilt_axis_angle"] = JobOption("Tilt axis angle (deg):", 90.0, 0.0, 180.0, 1.0 , "Nominal value for the angle of the tilt axis"); + joboptions["tilt_axis_angle"] = JobOption("Tilt axis angle (deg):", 85.0, 0.0, 180.0, 1.0 , "Nominal value for the tilt-axis rotation angle (positive is CCW from Y)"); joboptions["mtf_file"] = JobOption("MTF file:", (std::string)"","MTF file for the detector"); joboptions["do_tomo"] = JobOption("Import tomograms?", false, "Set this to Yes for importing tomogram directories from IMOD."); From 42240212bedbe1a6c5ec4fd0afb19e1e98d958a3 Mon Sep 17 00:00:00 2001 From: alisterburt Date: Thu, 23 Jun 2022 19:59:36 +0100 Subject: [PATCH 057/495] add description of prefix argument --- src/pipeline_jobs.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipeline_jobs.cpp b/src/pipeline_jobs.cpp index 355ba4ef1..8494eff3d 100644 --- a/src/pipeline_jobs.cpp +++ b/src/pipeline_jobs.cpp @@ -6074,7 +6074,7 @@ void RelionJob::initialiseTomoImportJob() joboptions["do_tiltseries"]= JobOption("Import tilt-series?", true, "Set this to Yes for importing tilt movies from SerialEM mdoc format metadata."); joboptions["movie_files"] = JobOption("Tilt image movie files:", (std::string)"frames/*.mrc","File pattern pointing to the raw movie files for the tilt images"); joboptions["mdoc_files"] = JobOption("mdoc files:", (std::string)"mdoc/*.mdoc","File pattern pointing to the mdoc files."); - joboptions["prefix"] = JobOption("Prefix:", (std::string)"","Prefix for XXX"); + joboptions["prefix"] = JobOption("Prefix:", (std::string)"","Optional prefix added to avoid tilt-series name collisions when dealing with multiple datasets."); joboptions["tilt_axis_angle"] = JobOption("Tilt axis angle (deg):", 85.0, 0.0, 180.0, 1.0 , "Nominal value for the tilt-axis rotation angle (positive is CCW from Y)"); joboptions["mtf_file"] = JobOption("MTF file:", (std::string)"","MTF file for the detector"); From 725c2290f132d9cfac5dc7aa517b1960f7bfc89d Mon Sep 17 00:00:00 2001 From: alisterburt Date: Thu, 23 Jun 2022 20:05:30 +0100 Subject: [PATCH 058/495] update IMOD job labels --- src/pipeline_jobs.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pipeline_jobs.cpp b/src/pipeline_jobs.cpp index 8494eff3d..99523360f 100644 --- a/src/pipeline_jobs.cpp +++ b/src/pipeline_jobs.cpp @@ -6327,10 +6327,10 @@ void RelionJob::initialiseTomoAlignTiltSeriesJob() if (default_location == NULL) default_location = default_wrapper; joboptions["imod_wrapper"] = JobOption("Alister Burt's IMOD/AreTomo wrapper:", std::string(default_location), "*", ".", "Location of Alister Burt's IMOD/Wrapper script; or just its executable name if in the path. You can control the default of this field by setting environment variable RELION_IMOD_WRAPPER_EXECUTABLE, or by editing the first few lines in src/gui_jobwindow.h and recompile the code. Note that Alister's script should find the executables to IMOD and AreTomo on its own. See Alister's documentation on how to configure this."); - joboptions["do_imod_fiducials"] = JobOption("Use IMOD:fiducials?", true, "Set to Yes to perform tilt series alignment using fiducials in IMOD."); - joboptions["fiducial_diameter"] = JobOption("Fiducial diameter (nm): ", 10, 1, 50, 1, "The diameter of the fiducials (in nm)"); + joboptions["do_imod_fiducials"] = JobOption("Use IMOD's fiducial based alignment?", true, "Set to Yes to perform tilt series alignment using fiducials in IMOD."); + joboptions["fiducial_diameter"] = JobOption("Fiducial diameter (nm): ", 10, 1, 20, 1, "The diameter of the fiducials (in nm)"); - joboptions["do_imod_patchtrack"] = JobOption("Use IMOD:patch-tracking?", false, "Set to Yes to perform tilt series alignment using patch-tracking in IMOD."); + joboptions["do_imod_patchtrack"] = JobOption("Use IMOD's patch-tracking for alignment?", false, "Set to Yes to perform tilt series alignment using patch-tracking in IMOD."); // TODO: check defaults with the experts joboptions["patch_size"] = JobOption("Patch size (in A): ", 10, 1, 50, 1, "The size of the patches in Angstrom."); joboptions["patch_overlap"] = JobOption("Patch overlap (%): ", 10, 0, 100, 10, "The overlap (0-100%) between the patches."); From 4ddbb103604a480f5124746fcc178c4d75cf2a1a Mon Sep 17 00:00:00 2001 From: alisterburt Date: Thu, 23 Jun 2022 20:05:55 +0100 Subject: [PATCH 059/495] change tiltseries to tilt-series in GUI browser --- src/gui_mainwindow.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gui_mainwindow.cpp b/src/gui_mainwindow.cpp index c4116b912..0298b2881 100644 --- a/src/gui_mainwindow.cpp +++ b/src/gui_mainwindow.cpp @@ -430,7 +430,7 @@ GuiMainWindow::GuiMainWindow(int w, int h, const char* title, FileName fn_pipe, nr_browse_tabs++; browse_grp[nr_browse_tabs] = new Fl_Group(WCOL0, 2, 550, 615-MENUHEIGHT); - browser->add("Align tiltseries"); + browser->add("Align tilt-series"); gui_jobwindows[nr_browse_tabs] = new JobWindow(); gui_jobwindows[nr_browse_tabs]->initialise(PROC_TOMO_ALIGN_TILTSERIES); browse_grp[nr_browse_tabs]->end(); From 2753dd1d7b5016d1da96d5885bca21495513ffc7 Mon Sep 17 00:00:00 2001 From: alisterburt Date: Fri, 24 Jun 2022 11:33:49 +0100 Subject: [PATCH 060/495] increment version number to 4.1-alpha-1 --- src/macros.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/macros.h b/src/macros.h index a4d54d02a..cca479f24 100644 --- a/src/macros.h +++ b/src/macros.h @@ -45,7 +45,7 @@ #ifndef MACROS_H #define MACROS_H -#define RELION_SHORT_VERSION "4.0-beta-2" +#define RELION_SHORT_VERSION "4.1-alpha-1" extern const char *g_RELION_VERSION; #include From a57b632257a33363ed6d1aab0b973932228a2058 Mon Sep 17 00:00:00 2001 From: scheres Date: Mon, 4 Jul 2022 15:06:44 +0100 Subject: [PATCH 061/495] minor request by Euan; some questions about gpu_ids in align tiltseries are pending --- src/align_tiltseries_runner.cpp | 29 +++++++++++++++++++++++++++-- src/align_tiltseries_runner.h | 10 +++++++++- src/align_tiltseries_runner_mpi.cpp | 2 +- src/gui_jobwindow.cpp | 4 ++++ src/pipeline_jobs.cpp | 14 ++++++++++++++ 5 files changed, 55 insertions(+), 4 deletions(-) diff --git a/src/align_tiltseries_runner.cpp b/src/align_tiltseries_runner.cpp index 05beea2e6..153f85bf8 100644 --- a/src/align_tiltseries_runner.cpp +++ b/src/align_tiltseries_runner.cpp @@ -42,6 +42,8 @@ void AlignTiltseriesRunner::read(int argc, char **argv, int rank) do_aretomo = parser.checkOption("--aretomo", "OR: Use AreTomo's alignment method"); aretomo_resolution = textToFloat(parser.getOption("--aretomo_resolution", "Maximum resolution (in A) to use in AreTomo alignments", "10")); aretomo_thickness = textToFloat(parser.getOption("--aretomo_thickness", "Thickness (in A) for AreTomo alignment", "2000")); + do_aretomo_tiltcorrect = parser.checkOption("--aretomo_tiltcorrect", "Specify to correct the tilt angle offset in the tomogram (AreTomo -TiltCor option; default=false)"); + gpu_ids = parser.getOption("--gpu", "Device ids for each MPI-thread, e.g 0:1:2:3", ""); patch_overlap = textToFloat(parser.getOption("--patch_overlap", "Overlap between the patches (in %)", "10")); patch_size = textToInteger(parser.getOption("--patch_size", "Patch size (in unbinned pixels)", "10")); @@ -141,7 +143,15 @@ void AlignTiltseriesRunner::initialise(bool is_leader) std::cout << do_at_most << " tomograms as specified in --do_at_most." << std::endl; } - if (verb > 0) + if (do_aretomo) + { + if (gpu_ids.length() > 0) + untangleDeviceIDs(gpu_ids, allThreadIDs); + else if (verb>0) + std::cout << "gpu-ids not specified, threads will automatically be mapped to devices."<< std::endl; + } + + if (verb > 0) { std::cout << " Using IMOD wrapper executable in: " << fn_imodwrapper_exe << std::endl; std::cout << " to align tilt series for the following tomograms: " << std::endl; @@ -184,7 +194,7 @@ void AlignTiltseriesRunner::run() } -void AlignTiltseriesRunner::executeImodWrapper(long idx_tomo) +void AlignTiltseriesRunner::executeImodWrapper(long idx_tomo, int rank) { RFLOAT angpix = tomogramSet.getPixelSize(idx_tomo); @@ -213,6 +223,21 @@ void AlignTiltseriesRunner::executeImodWrapper(long idx_tomo) command += " --tomogram-name " + tomogramSet.getTomogramName(idx_tomo); command += " --output-directory " + fn_out; + if (do_aretomo_tiltcorrect) command += " --tilt-angle-offset-correction"; + + if (gpu_ids.length() > 0) + { + if (rank >= allThreadIDs.size()) + REPORT_ERROR("ERROR: not enough MPI nodes specified for the GPU IDs."); + + command += " --gpu_ids "; + for (int igpu = 0; igpu < allThreadIDs[rank].size(); igpu++) + { + command += allThreadIDs[rank][igpu]; + if (igpu < allThreadIDs[rank].size()-1) command += ":"; + } + } + if (other_wrapper_args.length() > 0) command += " " + other_wrapper_args; diff --git a/src/align_tiltseries_runner.h b/src/align_tiltseries_runner.h index 0849dbbc6..88a1b743c 100644 --- a/src/align_tiltseries_runner.h +++ b/src/align_tiltseries_runner.h @@ -79,6 +79,14 @@ class AlignTiltseriesRunner // Alignment thickness in AreTomo RFLOAT aretomo_thickness; + // Perform tilt angle correction in AreTomo + bool do_aretomo_tiltcorrect; + + // Which GPU devices to use? + int devCount; + std::string gpu_ids; + std::vector < std::vector < std::string > > allThreadIDs; + // Additional gctf command line options std::string other_wrapper_args; @@ -105,7 +113,7 @@ class AlignTiltseriesRunner bool checkImodWrapperResults(long idx_tomo); // Execute CTFFIND for a single tomogram - void executeImodWrapper(long idx_tomo); + void executeImodWrapper(long idx_tomo, int rank = 0); // Find the etomo directives file (.edf) bool checkEtomoDirectiveFile(long idx_tomo, FileName &filename); diff --git a/src/align_tiltseries_runner_mpi.cpp b/src/align_tiltseries_runner_mpi.cpp index 231f5ad37..d057c9db7 100644 --- a/src/align_tiltseries_runner_mpi.cpp +++ b/src/align_tiltseries_runner_mpi.cpp @@ -60,7 +60,7 @@ void AlignTiltseriesRunnerMpi::run() if (pipeline_control_check_abort_job()) MPI_Abort(MPI_COMM_WORLD, RELION_EXIT_ABORTED); - executeImodWrapper(idx_tomograms[itomo]); + executeImodWrapper(idx_tomograms[itomo], node->rank); if (verb > 0 && itomo % barstep == 0) progress_bar(itomo); diff --git a/src/gui_jobwindow.cpp b/src/gui_jobwindow.cpp index d777630c8..7baa96397 100644 --- a/src/gui_jobwindow.cpp +++ b/src/gui_jobwindow.cpp @@ -2387,6 +2387,7 @@ void JobWindow::initialiseTomoImportWindow() current_y += STEPY/2; place("tilt_axis_angle", TOGGLE_DEACTIVATE); place("mtf_file", TOGGLE_DEACTIVATE); + place("flip_tiltseries_hand", TOGGLE_DEACTIVATE); group4->end(); guientries["do_tiltseries"].cb_menu_i(); // make default active @@ -2519,6 +2520,9 @@ void JobWindow::initialiseTomoAlignTiltseriesWindow() place("aretomo_resolution", TOGGLE_DEACTIVATE); place("aretomo_thickness", TOGGLE_DEACTIVATE); + place("aretomo_tiltcorrect", TOGGLE_DEACTIVATE); + + place("gpu_ids"); group2->end(); guientries["do_aretomo"].cb_menu_i(); // make default active diff --git a/src/pipeline_jobs.cpp b/src/pipeline_jobs.cpp index 8f7e9e7a2..271adfa70 100644 --- a/src/pipeline_jobs.cpp +++ b/src/pipeline_jobs.cpp @@ -6077,6 +6077,7 @@ void RelionJob::initialiseTomoImportJob() joboptions["prefix"] = JobOption("Prefix:", (std::string)"","Prefix for XXX"); joboptions["tilt_axis_angle"] = JobOption("Tilt axis angle (deg):", 90.0, 0.0, 180.0, 1.0 , "Nominal value for the angle of the tilt axis"); joboptions["mtf_file"] = JobOption("MTF file:", (std::string)"","MTF file for the detector"); + joboptions["flip_tiltseries_hand"] = JobOption("Invert Defocus Handedness?", false, "Specify Yes to flip the handedness of the defocus geometry (default = 1, the same as the tutorial dataset: EMPIAR-10164)"); joboptions["do_tomo"] = JobOption("Import tomograms?", true, "Set this to Yes for importing tomogram directories from IMOD."); joboptions["io_tomos"] = JobOption("Append to tomograms set: ", OUTNODE_TOMO_TOMOGRAMS, "", "Tomogram set STAR file (*.star)", "The imported tomograms will be output into this tomogram set. If any tomograms were already in this tomogram set, then the newly imported ones will be added to those."); @@ -6158,6 +6159,8 @@ bool RelionJob::getCommandsTomoImportJob(std::string &outputname, std::vector 0) + { + command += " --gpu-ids " + joboptions["gpu_ids"].getString(); + } } command += " --i " + joboptions["in_tiltseries"].getString(); Node node(joboptions["in_tiltseries"].getString(), joboptions["in_tiltseries"].node_type); From a5d040d5d7cc85b21beb74a3b478373d0a5de689 Mon Sep 17 00:00:00 2001 From: scheres Date: Mon, 4 Jul 2022 16:32:48 +0100 Subject: [PATCH 062/495] write out subtomos as 2D stacks for each particle --- src/jaz/tomography/particle_set.cpp | 16 +- src/jaz/tomography/particle_set.h | 3 +- src/jaz/tomography/programs/subtomo.cpp | 388 ++++++++++++------------ src/jaz/tomography/programs/subtomo.h | 3 +- src/metadata_label.h | 2 + 5 files changed, 208 insertions(+), 204 deletions(-) diff --git a/src/jaz/tomography/particle_set.cpp b/src/jaz/tomography/particle_set.cpp index cbacc7641..9719ba9d0 100644 --- a/src/jaz/tomography/particle_set.cpp +++ b/src/jaz/tomography/particle_set.cpp @@ -15,7 +15,20 @@ ParticleSet::ParticleSet() ParticleSet::ParticleSet(std::string filename, std::string motionFilename, bool verbose) { - optTable.read(filename, "optics"); + + if (genTable.read(filename,"general")) + { + genTable.getValueSafely(EMDL_TOMO_SUBTOMOGRAM_STACK2D, is_stack2d); + } + else + { + is_stack2d = false; + genTable.setIsList(true); + genTable.addObject(); + genTable.setValue(EMDL_TOMO_SUBTOMOGRAM_STACK2D, is_stack2d); + } + + optTable.read(filename, "optics"); partTable.read(filename, "particles"); if (!optTable.containsLabel(EMDL_TOMO_TILT_SERIES_PIXEL_SIZE)) @@ -399,6 +412,7 @@ void ParticleSet::write(const std::string& filename) const { std::ofstream ofs(filename); + genTable.write(ofs); optTable.write(ofs); partTable.write(ofs); } diff --git a/src/jaz/tomography/particle_set.h b/src/jaz/tomography/particle_set.h index 65057a18a..fc4977b1b 100644 --- a/src/jaz/tomography/particle_set.h +++ b/src/jaz/tomography/particle_set.h @@ -27,8 +27,9 @@ class ParticleSet ParticleSet(); ParticleSet(std::string filename, std::string motionFilename = "", bool verbose = true); - MetaDataTable partTable, optTable; + MetaDataTable partTable, optTable, genTable; + bool is_stack2d; bool hasMotion; std::vector motionTrajectories; diff --git a/src/jaz/tomography/programs/subtomo.cpp b/src/jaz/tomography/programs/subtomo.cpp index d58590db5..26792856e 100644 --- a/src/jaz/tomography/programs/subtomo.cpp +++ b/src/jaz/tomography/programs/subtomo.cpp @@ -47,6 +47,7 @@ void SubtomoProgram::readBasicParameters(IOParser& parser) boxSize = textToInteger(parser.getOption("--b", "Binned projection box size")); cropSize = textToInteger(parser.getOption("--crop", "Output box size", "-1")); binning = textToDouble(parser.getOption("--bin", "Binning factor", "1")); + do_stack2d = parser.checkOption("--stack2d", "Write out 2D stacks of cropped images for each particle, instead of pseudo-subtomograms"); rescale_coords = textToDouble(parser.getOption("--rescale_coords", "Rescale input particles by this factor", "1.")); write_multiplicity = parser.checkOption("--multi", "Write out multiplicity volumes"); SNR = textToDouble(parser.getOption("--SNR", "Assumed signal-to-noise ratio (negative means use a heuristic)", "-1")); @@ -144,6 +145,8 @@ void SubtomoProgram::run() BufferedImage sum_data, sum_weights; + if (do_stack2d) do_sum_all = false; + if (do_sum_all) { sum_data.resize(s3D,s3D,s3D); @@ -469,271 +472,254 @@ void SubtomoProgram::processTomograms( if (do_sum_all) omp_init_lock(&writelock); #pragma omp parallel for num_threads(outer_thread_num) - for (int p = 0; p < pc; p++) - { - const int th = omp_get_thread_num(); + for (int p = 0; p < pc; p++) { + const int th = omp_get_thread_num(); - if (verbosity > 0 && th == 0) - { - Log::updateProgress(p); - } + if (verbosity > 0 && th == 0) { + Log::updateProgress(p); + } - const ParticleIndex part_id = particles[t][p]; + const ParticleIndex part_id = particles[t][p]; - const std::string filenameRoot = getOutputFilename( - part_id, t, particleSet, tomogramSet); + const std::string filenameRoot = getOutputFilename( + part_id, t, particleSet, tomogramSet); - std::string outData = filenameRoot + "_data.mrc"; - std::string outWeight = filenameRoot + "_weights.mrc"; - std::string outCTF = filenameRoot + "_CTF2.mrc"; - std::string outDiv = filenameRoot + "_div.mrc"; - std::string outMulti = filenameRoot + "_multi.mrc"; - std::string outNrm = filenameRoot + "_data_nrm.mrc"; - std::string outWeightNrm = filenameRoot + "_CTF2_nrm.mrc"; + std::string outData = (do_stack2d) ? filenameRoot + "_stack2d.mrcs" : filenameRoot + "_data.mrc"; + std::string outWeight = filenameRoot + "_weights.mrc"; + std::string outCTF = filenameRoot + "_CTF2.mrc"; + std::string outDiv = filenameRoot + "_div.mrc"; + std::string outMulti = filenameRoot + "_multi.mrc"; + std::string outNrm = filenameRoot + "_data_nrm.mrc"; + std::string outWeightNrm = filenameRoot + "_CTF2_nrm.mrc"; - if (only_do_unfinished && ZIO::fileExists(outData)) - { - continue; - } - - const std::vector traj = particleSet.getTrajectoryInPixels( - part_id, fc, tomogram.optics.pixelSize, !apply_offsets); + if (only_do_unfinished && ZIO::fileExists(outData)) { + continue; + } - if (!tomogram.isVisibleAtAll(traj, s2D / 2.0)) - { - continue; - } + const std::vector traj = particleSet.getTrajectoryInPixels( + part_id, fc, tomogram.optics.pixelSize, !apply_offsets); - const std::vector isVisible = tomogram.determineVisiblity(traj, s2D / 2.0); + if (!tomogram.isVisibleAtAll(traj, s2D / 2.0)) { + continue; + } - std::vector projCut(fc), projPart(fc); + const std::vector isVisible = tomogram.determineVisiblity(traj, s2D / 2.0); - BufferedImage particleStack = BufferedImage(sh2D,s2D,fc); - BufferedImage weightStack(sh2D,s2D,fc); + std::vector projCut(fc), projPart(fc); - TomoExtraction::extractAt3D_Fourier( - tomogram.stack, s02D, binning, tomogram, traj, isVisible, - particleStack, projCut, inner_thread_num, do_circle_precrop); + BufferedImage particleStack = BufferedImage(sh2D, s2D, fc); + BufferedImage weightStack(sh2D, s2D, fc); - if (!do_ctf) weightStack.fill(1.f); + TomoExtraction::extractAt3D_Fourier( + tomogram.stack, s02D, binning, tomogram, traj, isVisible, + particleStack, projCut, inner_thread_num, do_circle_precrop); + if (do_stack2d) + { + BufferedImage particlesRS = NewStackHelper::inverseFourierTransformStack(particleStack); - const int og = particleSet.getOpticsGroup(part_id); + particlesRS.write(outData, binnedPixelSize, write_float16); + } + else + { - const BufferedImage* gammaOffset = - aberrationsCache.hasSymmetrical? &aberrationsCache.symmetrical[og] : 0; + if (!do_ctf) weightStack.fill(1.f); - for (int f = 0; f < fc; f++) - { - if (!isVisible[f]) continue; - d3Matrix A; + const int og = particleSet.getOpticsGroup(part_id); - if (apply_orientations) - { - A = particleSet.getMatrix3x3(part_id); - } - else - { - A = particleSet.getSubtomogramMatrix(part_id); - } + const BufferedImage *gammaOffset = + aberrationsCache.hasSymmetrical ? &aberrationsCache.symmetrical[og] : 0; - projPart[f] = projCut[f] * d4Matrix(A); - - if (do_ctf) - { - const d3Vector pos = (apply_offsets) ? particleSet.getPosition(part_id) : particleSet.getParticleCoord(part_id); + for (int f = 0; f < fc; f++) { + if (!isVisible[f]) continue; - CTF ctf = tomogram.getCtf(f, pos); - BufferedImage ctfImg(sh2D, s2D); - ctf.draw(s2D, s2D, binnedPixelSize, gammaOffset, &ctfImg(0,0,0)); + d3Matrix A; - const float sign = flip_value? -1.f : 1.f; + if (apply_orientations) { + A = particleSet.getMatrix3x3(part_id); + } else { + A = particleSet.getSubtomogramMatrix(part_id); + } - for (int y = 0; y < s2D; y++) - for (int x = 0; x < xRanges(y,f); x++) - { - const double c = ctfImg(x,y) * doseWeights(x,y,f); + projPart[f] = projCut[f] * d4Matrix(A); - particleStack(x,y,f) *= sign * c; - weightStack(x,y,f) = c * c; - } - } - } + if (do_ctf) { + const d3Vector pos = (apply_offsets) ? particleSet.getPosition(part_id) + : particleSet.getParticleCoord(part_id); - aberrationsCache.correctObservations(particleStack, og); + CTF ctf = tomogram.getCtf(f, pos); + BufferedImage ctfImg(sh2D, s2D); + ctf.draw(s2D, s2D, binnedPixelSize, gammaOffset, &ctfImg(0, 0, 0)); - if (do_whiten) - { - particleStack *= noiseWeights; - weightStack *= noiseWeights; - } + const float sign = flip_value ? -1.f : 1.f; - const int boundary = (boxSize - cropSize) / 2; + for (int y = 0; y < s2D; y++) + for (int x = 0; x < xRanges(y, f); x++) { + const double c = ctfImg(x, y) * doseWeights(x, y, f); - if (do_gridding_precorrection || do_circle_crop) - { - BufferedImage particlesRS; + particleStack(x, y, f) *= sign * c; + weightStack(x, y, f) = c * c; + } + } + } - particlesRS = NewStackHelper::inverseFourierTransformStack(particleStack); + aberrationsCache.correctObservations(particleStack, og); - if (do_circle_crop) - { - const double crop_boundary = do_narrow_circle_crop? boundary : 0.0; - TomoExtraction::cropCircle(particlesRS, crop_boundary, 5, num_threads); - } + if (do_whiten) { + particleStack *= noiseWeights; + weightStack *= noiseWeights; + } - if (do_gridding_precorrection) - { - TomoExtraction::griddingPreCorrect(particlesRS, boundary, num_threads); - } + const int boundary = (boxSize - cropSize) / 2; - particleStack = NewStackHelper::FourierTransformStack(particlesRS); - } + if (do_gridding_precorrection || do_circle_crop) { + BufferedImage particlesRS; - BufferedImage dataImgFS(sh3D,s3D,s3D); - dataImgFS.fill(fComplex(0.0, 0.0)); + particlesRS = NewStackHelper::inverseFourierTransformStack(particleStack); - BufferedImage ctfImgFS(sh3D,s3D,s3D), - dataImgRS(s3D,s3D,s3D), dataImgDivRS(s3D,s3D,s3D), - multiImageFS(sh3D,s3D,s3D); + if (do_circle_crop) { + const double crop_boundary = do_narrow_circle_crop ? boundary : 0.0; + TomoExtraction::cropCircle(particlesRS, crop_boundary, 5, num_threads); + } - ctfImgFS.fill(0.0); - dataImgRS.fill(0.0); - dataImgDivRS.fill(0.0); + if (do_gridding_precorrection) { + TomoExtraction::griddingPreCorrect(particlesRS, boundary, num_threads); + } - for (int f = 0; f < fc; f++) - { - if (isVisible[f]) - { - FourierBackprojection::backprojectSlice_forward_with_multiplicity( - &xRanges(0,f), - particleStack.getSliceRef(f), - weightStack.getSliceRef(f), - projPart[f] * relative_box_scale, - dataImgFS, - ctfImgFS, - multiImageFS); - } - } + particleStack = NewStackHelper::FourierTransformStack(particlesRS); + } - Centering::shiftInSitu(dataImgFS); + BufferedImage dataImgFS(sh3D, s3D, s3D); + dataImgFS.fill(fComplex(0.0, 0.0)); + + BufferedImage ctfImgFS(sh3D, s3D, s3D), + dataImgRS(s3D, s3D, s3D), dataImgDivRS(s3D, s3D, s3D), + multiImageFS(sh3D, s3D, s3D); + + ctfImgFS.fill(0.0); + dataImgRS.fill(0.0); + dataImgDivRS.fill(0.0); + + for (int f = 0; f < fc; f++) { + if (isVisible[f]) { + FourierBackprojection::backprojectSlice_forward_with_multiplicity( + &xRanges(0, f), + particleStack.getSliceRef(f), + weightStack.getSliceRef(f), + projPart[f] * relative_box_scale, + dataImgFS, + ctfImgFS, + multiImageFS); + } + } - // correct FT scale after the implicit cropping: + Centering::shiftInSitu(dataImgFS); - if (s3D != s2D) - { - dataImgFS *= (float) sqrt(s2D / (double) s3D); - } + // correct FT scale after the implicit cropping: - FFT::inverseFourierTransform(dataImgFS, dataImgRS, FFT::Both); + if (s3D != s2D) { + dataImgFS *= (float) sqrt(s2D / (double) s3D); + } - if (do_cone_weight) - { - FFT::FourierTransform(dataImgRS, dataImgFS); + FFT::inverseFourierTransform(dataImgFS, dataImgRS, FFT::Both); - d3Matrix R = particleSet.getMatrix3x3(part_id); + if (do_cone_weight) { + FFT::FourierTransform(dataImgRS, dataImgFS); - for (int z = 0; z < s3D; z++) - for (int y = 0; y < s3D; y++) - for (int x = 0; x < sh3D; x++) - { - const d3Vector p0( - x, - y < s3D/2? y : y - s3D, - z < s3D/2? z : z - s3D); + d3Matrix R = particleSet.getMatrix3x3(part_id); - const d3Vector p = R * p0; + for (int z = 0; z < s3D; z++) + for (int y = 0; y < s3D; y++) + for (int x = 0; x < sh3D; x++) { + const d3Vector p0( + x, + y < s3D / 2 ? y : y - s3D, + z < s3D / 2 ? z : z - s3D); - const double rho = sqrt(p.x*p.x + p.y*p.y); - const double t = rho / (std::abs(p.z) * cone_slope + cone_sig0); + const d3Vector p = R * p0; - const double m = 1.0 - exp(-0.5*t*t); + const double rho = sqrt(p.x * p.x + p.y * p.y); + const double t = rho / (std::abs(p.z) * cone_slope + cone_sig0); - dataImgFS(x,y,z) *= m; - ctfImgFS(x,y,z) *= m; - multiImageFS(x,y,z) *= m; // apply to both multiplicity and weight? - } + const double m = 1.0 - exp(-0.5 * t * t); - FFT::inverseFourierTransform(dataImgFS, dataImgRS); - } + dataImgFS(x, y, z) *= m; + ctfImgFS(x, y, z) *= m; + multiImageFS(x, y, z) *= m; // apply to both multiplicity and weight? + } - // What if we didn't? The 2D image is already tapered. - //Reconstruction::taper(dataImgRS, taper, do_center, inner_thread_num); + FFT::inverseFourierTransform(dataImgFS, dataImgRS); + } - if (do_sum_all) - { - omp_set_lock(&writelock); + // What if we didn't? The 2D image is already tapered. + //Reconstruction::taper(dataImgRS, taper, do_center, inner_thread_num); - sum_data += dataImgRS; - sum_weights += ctfImgFS; + if (do_sum_all) { + omp_set_lock(&writelock); - omp_unset_lock(&writelock); - } + sum_data += dataImgRS; + sum_weights += ctfImgFS; - if (do_not_write_any) continue; + omp_unset_lock(&writelock); + } + if (do_not_write_any) continue; - dataImgRS.write(outData, binnedPixelSize, write_float16); - if (write_combined) - { - BufferedImage ctfAndMultiplicity(sh3D,s3D,2*s3D); - ctfAndMultiplicity.getSlabRef(0,s3D).copyFrom(ctfImgFS); - ctfAndMultiplicity.getSlabRef(s3D,s3D).copyFrom(multiImageFS); + dataImgRS.write(outData, binnedPixelSize, write_float16); - ctfAndMultiplicity.write(outWeight, 1.0 / binnedPixelSize, write_float16); - } + if (write_combined) { + BufferedImage ctfAndMultiplicity(sh3D, s3D, 2 * s3D); + ctfAndMultiplicity.getSlabRef(0, s3D).copyFrom(ctfImgFS); + ctfAndMultiplicity.getSlabRef(s3D, s3D).copyFrom(multiImageFS); - if (write_ctf) - { - Centering::fftwHalfToHumanFull(ctfImgFS).write(outCTF, 1.0 / binnedPixelSize, write_float16); - } + ctfAndMultiplicity.write(outWeight, 1.0 / binnedPixelSize, write_float16); + } - if (write_multiplicity) - { - Centering::fftwHalfToHumanFull(multiImageFS).write(outMulti, 1.0 / binnedPixelSize, write_float16); - } + if (write_ctf) { + Centering::fftwHalfToHumanFull(ctfImgFS).write(outCTF, 1.0 / binnedPixelSize, write_float16); + } - if (write_normalised) - { - BufferedImage ctfImgFSnrm = ctfImgFS; - BufferedImage dataImgCorrFS; + if (write_multiplicity) { + Centering::fftwHalfToHumanFull(multiImageFS).write(outMulti, 1.0 / binnedPixelSize, write_float16); + } - FFT::FourierTransform(dataImgRS, dataImgCorrFS, FFT::Both); + if (write_normalised) { + BufferedImage ctfImgFSnrm = ctfImgFS; + BufferedImage dataImgCorrFS; - for (long int i = 0; i < ctfImgFSnrm.getSize(); i++) - { - const float n = multiImageFS[i]; - ctfImgFSnrm[i] = n > 0.f? ctfImgFS[i] / n : 0.f; - dataImgCorrFS[i] = n > 0.f? dataImgCorrFS[i] / n : fComplex(0.f,0.f); - } + FFT::FourierTransform(dataImgRS, dataImgCorrFS, FFT::Both); - FFT::inverseFourierTransform(dataImgCorrFS, dataImgDivRS, FFT::Both); + for (long int i = 0; i < ctfImgFSnrm.getSize(); i++) { + const float n = multiImageFS[i]; + ctfImgFSnrm[i] = n > 0.f ? ctfImgFS[i] / n : 0.f; + dataImgCorrFS[i] = n > 0.f ? dataImgCorrFS[i] / n : fComplex(0.f, 0.f); + } - dataImgDivRS.write(outNrm, binnedPixelSize, write_float16); - Centering::fftwHalfToHumanFull(ctfImgFSnrm).write(outWeightNrm, 1.0 / binnedPixelSize, write_float16); - } + FFT::inverseFourierTransform(dataImgCorrFS, dataImgDivRS, FFT::Both); - if (write_divided) - { - if (SNR > 0.0) - { - Reconstruction::ctfCorrect3D_Wiener( - dataImgRS, ctfImgFS, dataImgDivRS, - 1.0 / SNR, inner_thread_num); - } - else - { - Reconstruction::ctfCorrect3D_heuristic( - dataImgRS, ctfImgFS, dataImgDivRS, - 0.001, inner_thread_num); - } + dataImgDivRS.write(outNrm, binnedPixelSize, write_float16); + Centering::fftwHalfToHumanFull(ctfImgFSnrm).write(outWeightNrm, 1.0 / binnedPixelSize, + write_float16); + } - Reconstruction::taper(dataImgDivRS, taper, do_center, inner_thread_num); - dataImgDivRS.write(outDiv, binnedPixelSize, write_float16); - } - } + if (write_divided) { + if (SNR > 0.0) { + Reconstruction::ctfCorrect3D_Wiener( + dataImgRS, ctfImgFS, dataImgDivRS, + 1.0 / SNR, inner_thread_num); + } else { + Reconstruction::ctfCorrect3D_heuristic( + dataImgRS, ctfImgFS, dataImgDivRS, + 0.001, inner_thread_num); + } + + Reconstruction::taper(dataImgDivRS, taper, do_center, inner_thread_num); + dataImgDivRS.write(outDiv, binnedPixelSize, write_float16); + } + } + } // end if !do_stack2d if (verbosity > 0) { diff --git a/src/jaz/tomography/programs/subtomo.h b/src/jaz/tomography/programs/subtomo.h index dd1ff745c..db57b9b6e 100644 --- a/src/jaz/tomography/programs/subtomo.h +++ b/src/jaz/tomography/programs/subtomo.h @@ -42,7 +42,8 @@ class SubtomoProgram bool flip_value, - diag, + diag, + do_stack2d, do_whiten, do_center, do_rotate, diff --git a/src/metadata_label.h b/src/metadata_label.h index 145a59eb5..42eae4be9 100644 --- a/src/metadata_label.h +++ b/src/metadata_label.h @@ -628,6 +628,7 @@ enum EMDLabel EMDL_TOMO_SUBTOMOGRAM_ROT, EMDL_TOMO_SUBTOMOGRAM_TILT, EMDL_TOMO_SUBTOMOGRAM_PSI, + EMDL_TOMO_SUBTOMOGRAM_STACK2D, EMDL_TOMO_SUBTOMOGRAM_BINNING, EMDL_TOMO_TOMOGRAM_BINNING, EMDL_TOMO_PARTICLE_NAME, @@ -1329,6 +1330,7 @@ class StaticInitialization EMDL::addLabel(EMDL_TOMO_SUBTOMOGRAM_ROT, EMDL_DOUBLE, "rlnTomoSubtomogramRot", "First Euler angle of a subtomogram (rot, in degrees)"); EMDL::addLabel(EMDL_TOMO_SUBTOMOGRAM_TILT, EMDL_DOUBLE, "rlnTomoSubtomogramTilt", "Second Euler angle of a subtomogram (tilt, in degrees)"); EMDL::addLabel(EMDL_TOMO_SUBTOMOGRAM_PSI, EMDL_DOUBLE, "rlnTomoSubtomogramPsi", "Third Euler angle of a subtomogram (psi, in degrees)"); + EMDL::addLabel(EMDL_TOMO_SUBTOMOGRAM_STACK2D, EMDL_BOOL, "rlnTomoSubTomosAre2DStacks", "This flag is set to true if subtomograms are saved as 2D image stacks"); EMDL::addLabel(EMDL_TOMO_SUBTOMOGRAM_BINNING, EMDL_DOUBLE, "rlnTomoSubtomogramBinning", "Binning level of a subtomogram"); EMDL::addLabel(EMDL_TOMO_TOMOGRAM_BINNING, EMDL_DOUBLE, "rlnTomoTomogramBinning", "Binning level of a reconstructed tomogram"); EMDL::addLabel(EMDL_TOMO_PARTICLE_NAME, EMDL_STRING, "rlnTomoParticleName", "Name of each individual particle"); From e0afca7f7a41de43bd59284cb133983347a08a40 Mon Sep 17 00:00:00 2001 From: scheres Date: Mon, 4 Jul 2022 16:33:43 +0100 Subject: [PATCH 063/495] typos gpu_ids instead of gpu-ids --- src/pipeline_jobs.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipeline_jobs.cpp b/src/pipeline_jobs.cpp index 271adfa70..e2f527bf2 100644 --- a/src/pipeline_jobs.cpp +++ b/src/pipeline_jobs.cpp @@ -6395,7 +6395,7 @@ bool RelionJob::getCommandsTomoAlignTiltSeriesJob(std::string &outputname, std:: } if (joboptions["gpu_ids"].getString().length() > 0) { - command += " --gpu-ids " + joboptions["gpu_ids"].getString(); + command += " --gpu_ids " + joboptions["gpu_ids"].getString(); } } command += " --i " + joboptions["in_tiltseries"].getString(); From 2119dbeb7bff735645e6d4ba3cc17619381aeb97 Mon Sep 17 00:00:00 2001 From: scheres Date: Mon, 4 Jul 2022 16:34:36 +0100 Subject: [PATCH 064/495] reverse type in align_tiltseries --- src/align_tiltseries_runner.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/align_tiltseries_runner.cpp b/src/align_tiltseries_runner.cpp index 153f85bf8..24a1244ef 100644 --- a/src/align_tiltseries_runner.cpp +++ b/src/align_tiltseries_runner.cpp @@ -148,7 +148,7 @@ void AlignTiltseriesRunner::initialise(bool is_leader) if (gpu_ids.length() > 0) untangleDeviceIDs(gpu_ids, allThreadIDs); else if (verb>0) - std::cout << "gpu-ids not specified, threads will automatically be mapped to devices."<< std::endl; + std::cout << "--gpu_ids not specified, threads will automatically be mapped to devices."<< std::endl; } if (verb > 0) @@ -230,7 +230,7 @@ void AlignTiltseriesRunner::executeImodWrapper(long idx_tomo, int rank) if (rank >= allThreadIDs.size()) REPORT_ERROR("ERROR: not enough MPI nodes specified for the GPU IDs."); - command += " --gpu_ids "; + command += " --gpu-ids "; for (int igpu = 0; igpu < allThreadIDs[rank].size(); igpu++) { command += allThreadIDs[rank][igpu]; From 64420569f83d6b71423822d4ee4886b6fd3740a1 Mon Sep 17 00:00:00 2001 From: scheres Date: Mon, 4 Jul 2022 17:23:44 +0100 Subject: [PATCH 065/495] adjust namings of 2D stacks --- src/jaz/tomography/particle_set.cpp | 3 ++- src/jaz/tomography/programs/subtomo.cpp | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/jaz/tomography/particle_set.cpp b/src/jaz/tomography/particle_set.cpp index 9719ba9d0..816a284bb 100644 --- a/src/jaz/tomography/particle_set.cpp +++ b/src/jaz/tomography/particle_set.cpp @@ -459,7 +459,8 @@ void ParticleSet::writeTrajectories(const std::string &filename) const void ParticleSet::setImageFileNames(std::string data, std::string weight, ParticleIndex particle_id) { partTable.setValue(EMDL_IMAGE_NAME, data, particle_id.value); - partTable.setValue(EMDL_CTF_IMAGE, weight, particle_id.value); + if (weight != "") + partTable.setValue(EMDL_CTF_IMAGE, weight, particle_id.value); } d3Vector ParticleSet::getParticleOffset(ParticleIndex particle_id) const diff --git a/src/jaz/tomography/programs/subtomo.cpp b/src/jaz/tomography/programs/subtomo.cpp index 26792856e..3b627c2dd 100644 --- a/src/jaz/tomography/programs/subtomo.cpp +++ b/src/jaz/tomography/programs/subtomo.cpp @@ -297,6 +297,7 @@ void SubtomoProgram::writeParticleSet( ParticleSet copy = particleSet; copy.clearParticles(); + copy.is_stack2d = do_stack2d; int particles_removed = 0; @@ -325,8 +326,8 @@ void SubtomoProgram::writeParticleSet( const std::string filenameRoot = getOutputFilename( part_id, t, particleSet, tomogramSet); - std::string outData = filenameRoot + "_data.mrc"; - std::string outWeight = filenameRoot + "_weights.mrc"; + std::string outData = (do_stack2d) ? filenameRoot + "_stack2d.mrcs" : filenameRoot + "_data.mrc"; + std::string outWeight = (do_stack2d) ? "" : filenameRoot + "_weights.mrc"; copy.setImageFileNames(outData, outWeight, new_id); From 870e1b91df2b5cdb28823881888e635a7e073a80 Mon Sep 17 00:00:00 2001 From: scheres Date: Mon, 4 Jul 2022 17:24:27 +0100 Subject: [PATCH 066/495] started with putting tomogramSet in .... --- src/ml_optimiser.cpp | 14 ++++++++++++-- src/ml_optimiser.h | 13 ++++++++++++- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/ml_optimiser.cpp b/src/ml_optimiser.cpp index 84242eaa9..4e1a2e95e 100644 --- a/src/ml_optimiser.cpp +++ b/src/ml_optimiser.cpp @@ -516,7 +516,7 @@ if(do_gpu) // We read input optimiser set to create the output one fn_OS = parser.getOption("--ios", "Input tomo optimiser set file. It is used to set --i, --ref or --solvent_mask if they are not provided. Updated output optimiser set is created.", ""); - if (fn_OS != "") + if (fn_OS != "") { optimisationSet.read(fn_OS); } @@ -552,7 +552,8 @@ void MlOptimiser::parseInitial(int argc, char **argv) // General optimiser I/O stuff int general_section = parser.addSection("General options"); - fn_data = parser.getOption("--i", "Input images (in a star-file)", ""); + fn_data = parser.getOption("--i", "Input particles (in a star-file)", ""); + fn_tomo = parser.getOption("--tomograms", "Star file with the tomograms (in a star-file)", ""); fn_OS = parser.getOption("--ios", "Input tomo optimiser set file. It is used to set --i, --ref or --solvent_mask if they are not provided. Updated output optimiser set is created.", ""); fn_out = parser.getOption("--o", "Output rootname", ""); nr_iter = textToInteger(parser.getOption("--iter", "Maximum number of iterations to perform", "-1")); @@ -590,6 +591,11 @@ void MlOptimiser::parseInitial(int argc, char **argv) if (!optimisationSet.getValue(EMDL_TOMO_PARTICLES_FILE_NAME, fn_data)) REPORT_ERROR("No particles filename was found in file " + fn_OS); } + if (fn_tomo == "") + { + if (!optimisationSet.getValue(EMDL_TOMO_TOMOGRAMS_FILE_NAME, fn_tomo)) + REPORT_ERROR("No tomograms filename was found in file " + fn_OS); + } if (fn_ref == "None") { if (!optimisationSet.getValue(EMDL_TOMO_REFERENCE_MAP_1_FILE_NAME, fn_ref)) @@ -1122,6 +1128,8 @@ void MlOptimiser::read(FileName fn_in, int rank, bool do_prevent_preread) do_center_classes = false; if (!MD.getValue(EMDL_OPTIMISER_DO_AUTO_SAMPLING, do_auto_sampling)) do_auto_sampling = false; + if (!MD.getValue(EMDL_TOMO_TOMOGRAMS_FILE_NAME, fn_tomo)) + fn_tomo = ""; // Initialise some stuff for first-iteration only (not relevant here...) do_calculate_initial_sigma_noise = false; @@ -1248,6 +1256,7 @@ void MlOptimiser::write(bool do_write_sampling, bool do_write_data, bool do_writ MD.setValue(EMDL_OPTIMISER_MODEL_STARFILE, fn_model); } MD.setValue(EMDL_OPTIMISER_DATA_STARFILE, fn_data); + MD.setValue(EMDL_TOMO_TOMOGRAMS_FILE_NAME, fn_tomo); MD.setValue(EMDL_OPTIMISER_SAMPLING_STARFILE, fn_sampling); MD.setValue(EMDL_OPTIMISER_ITERATION_NO, iter); MD.setValue(EMDL_OPTIMISER_NR_ITERATIONS, nr_iter); @@ -1373,6 +1382,7 @@ void MlOptimiser::write(bool do_write_sampling, bool do_write_data, bool do_writ if (do_join_random_halves && !optimisationSet.isEmpty()) { optimisationSet.setValue(EMDL_TOMO_PARTICLES_FILE_NAME, fn_root + "_data.star"); + optimisationSet.setValue(EMDL_TOMO_TOMOGRAMS_FILE_NAME, fn_tomo); optimisationSet.setValue(EMDL_TOMO_REFERENCE_MAP_1_FILE_NAME, fn_root2 + "_half1_class001_unfil.mrc"); optimisationSet.setValue(EMDL_TOMO_REFERENCE_MAP_2_FILE_NAME, fn_root2 + "_half2_class001_unfil.mrc"); optimisationSet.setValue(EMDL_TOMO_REFERENCE_MASK_FILE_NAME, fn_mask); diff --git a/src/ml_optimiser.h b/src/ml_optimiser.h index 189810776..1e384852d 100644 --- a/src/ml_optimiser.h +++ b/src/ml_optimiser.h @@ -40,6 +40,11 @@ #include "src/helix.h" #include "src/local_symmetry.h" #include "src/acc/settings.h" +#include +#include +#include +#include +#include #define ML_SIGNIFICANT_WEIGHT 1.e-8 #define METADATA_LINE_LENGTH METADATA_LINE_LENGTH_ALL @@ -146,7 +151,13 @@ class MlOptimiser // Optimiser set table for subtomo MetaDataTable optimisationSet; - // Generate a 3D model from 2D particles de novo? + // For relion-4.1 tomogram-processing as 2D stacks + TomogramSet tomogramSet; + ParticleSet particleSet; + // FileName for tomogramSet + FileName fn_tomo; + + // Generate a 3D model from 2D particles de novo? bool is_3d_model; //User-specified pixel size for the reference From 475d7ba6dc70ffccad2e4d932b757a75774465ba Mon Sep 17 00:00:00 2001 From: scheres Date: Tue, 5 Jul 2022 08:50:31 +0100 Subject: [PATCH 067/495] --gpu for align tiltseries runner --- src/pipeline_jobs.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipeline_jobs.cpp b/src/pipeline_jobs.cpp index 60885d140..98d16b47d 100644 --- a/src/pipeline_jobs.cpp +++ b/src/pipeline_jobs.cpp @@ -6395,7 +6395,7 @@ bool RelionJob::getCommandsTomoAlignTiltSeriesJob(std::string &outputname, std:: } if (joboptions["gpu_ids"].getString().length() > 0) { - command += " --gpu_ids " + joboptions["gpu_ids"].getString(); + command += " --gpu " + joboptions["gpu_ids"].getString(); } } command += " --i " + joboptions["in_tiltseries"].getString(); From 5404dbd88ac55c58f0f971da5aaee6fc526f1686 Mon Sep 17 00:00:00 2001 From: scheres Date: Tue, 5 Jul 2022 10:01:47 +0100 Subject: [PATCH 068/495] added read function to particleSet --- src/jaz/tomography/particle_set.cpp | 13 ++++++++++++- src/jaz/tomography/particle_set.h | 3 ++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/jaz/tomography/particle_set.cpp b/src/jaz/tomography/particle_set.cpp index 816a284bb..d92546bbc 100644 --- a/src/jaz/tomography/particle_set.cpp +++ b/src/jaz/tomography/particle_set.cpp @@ -14,6 +14,12 @@ ParticleSet::ParticleSet() {} ParticleSet::ParticleSet(std::string filename, std::string motionFilename, bool verbose) +{ + if (!read(filename, motionFilename, verbose)) + REPORT_ERROR("ERROR: there are no particles in " + filename); +} + +bool ParticleSet::read(std::string filename, std::string motionFilename, bool verbose) { if (genTable.read(filename,"general")) @@ -30,7 +36,10 @@ ParticleSet::ParticleSet(std::string filename, std::string motionFilename, bool optTable.read(filename, "optics"); partTable.read(filename, "particles"); - + + long int pc = partTable.numberOfObjects(); + if (pc == 0) return false; + if (!optTable.containsLabel(EMDL_TOMO_TILT_SERIES_PIXEL_SIZE)) { REPORT_ERROR("ParticleSet::ParticleSet: " @@ -79,6 +88,8 @@ ParticleSet::ParticleSet(std::string filename, std::string motionFilename, bool { motionTrajectories = Trajectory::read(motionFilename, *this); } + + return true; } void ParticleSet::reserve(int particleNumber) diff --git a/src/jaz/tomography/particle_set.h b/src/jaz/tomography/particle_set.h index fc4977b1b..6ac8f6adb 100644 --- a/src/jaz/tomography/particle_set.h +++ b/src/jaz/tomography/particle_set.h @@ -26,7 +26,8 @@ class ParticleSet ParticleSet(); ParticleSet(std::string filename, std::string motionFilename = "", bool verbose = true); - + bool read(std::string filename, std::string motionFilename = "", bool verbose = true); + MetaDataTable partTable, optTable, genTable; bool is_stack2d; From 077c4c066c1fc999a100040375e673a811ebeac1 Mon Sep 17 00:00:00 2001 From: scheres Date: Tue, 5 Jul 2022 13:08:32 +0100 Subject: [PATCH 069/495] added particleSet and tomogramSet to exp_model, also modified other files to keep compiling --- src/apps/particle_reposition.cpp | 35 ++- src/class_ranker.cpp | 4 +- src/exp_model.cpp | 245 +++++++++----------- src/exp_model.h | 60 ++--- src/flex_analyser.cpp | 2 +- src/ml_optimiser.cpp | 368 ++++++++++++++++--------------- src/ml_optimiser.h | 11 +- src/ml_optimiser_mpi.cpp | 2 +- src/particle_subtractor.cpp | 67 +++--- 9 files changed, 386 insertions(+), 408 deletions(-) diff --git a/src/apps/particle_reposition.cpp b/src/apps/particle_reposition.cpp index db0f0d7b1..62d91980c 100644 --- a/src/apps/particle_reposition.cpp +++ b/src/apps/particle_reposition.cpp @@ -157,7 +157,6 @@ class particle_reposition_parameters bool found_one = false; for (long int part_id = 0; part_id < optimiser.mydata.numberOfParticles(); part_id++) { - long int ori_img_id = optimiser.mydata.particles[part_id].images[0].id; int optics_group = optimiser.mydata.getOpticsGroup(part_id, 0); RFLOAT my_pixel_size = optimiser.mydata.getImagePixelSize(part_id, 0); int my_image_size = optimiser.mydata.getOpticsImageSize(optics_group); @@ -166,7 +165,7 @@ class particle_reposition_parameters REPORT_ERROR("ERROR: subtract code has only been validated with same pixel size for particles and micrographs... Sorry!"); FileName fn_mic2; - optimiser.mydata.MDimg.getValue(EMDL_MICROGRAPH_NAME, fn_mic2, ori_img_id); + optimiser.mydata.MDimg.getValue(EMDL_MICROGRAPH_NAME, fn_mic2, part_id); FileName fn_mic2_pre, fn_mic2_jobnr, fn_mic2_post; decomposePipelineFileName(fn_mic2, fn_mic2_pre, fn_mic2_jobnr, fn_mic2_post); @@ -197,23 +196,23 @@ class particle_reposition_parameters MDcoord.addObject(); - MDcoord.setObject(optimiser.mydata.MDimg.getObject(ori_img_id)); + MDcoord.setObject(optimiser.mydata.MDimg.getObject(part_id)); MDcoord.setValue(EMDL_MICROGRAPH_NAME,fn_mic_out); - optimiser.mydata.MDimg.getValue(EMDL_IMAGE_COORD_X, xcoord, ori_img_id); - optimiser.mydata.MDimg.getValue(EMDL_IMAGE_COORD_Y, ycoord, ori_img_id); + optimiser.mydata.MDimg.getValue(EMDL_IMAGE_COORD_X, xcoord, part_id); + optimiser.mydata.MDimg.getValue(EMDL_IMAGE_COORD_Y, ycoord, part_id); if (optimiser.mymodel.ref_dim == 3) { - optimiser.mydata.MDimg.getValue(EMDL_ORIENT_ROT, rot, ori_img_id); - optimiser.mydata.MDimg.getValue(EMDL_ORIENT_TILT, tilt, ori_img_id); + optimiser.mydata.MDimg.getValue(EMDL_ORIENT_ROT, rot, part_id); + optimiser.mydata.MDimg.getValue(EMDL_ORIENT_TILT, tilt, part_id); } - optimiser.mydata.MDimg.getValue(EMDL_ORIENT_PSI, psi, ori_img_id); - optimiser.mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_X_ANGSTROM, XX(offsets), ori_img_id); - optimiser.mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_Y_ANGSTROM, YY(offsets), ori_img_id); + optimiser.mydata.MDimg.getValue(EMDL_ORIENT_PSI, psi, part_id); + optimiser.mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_X_ANGSTROM, XX(offsets), part_id); + optimiser.mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_Y_ANGSTROM, YY(offsets), part_id); if (optimiser.mymodel.data_dim == 3) { - optimiser.mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_Z_ANGSTROM, ZZ(offsets), ori_img_id); - optimiser.mydata.MDimg.getValue(EMDL_IMAGE_COORD_Z, zcoord, ori_img_id); + optimiser.mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_Z_ANGSTROM, ZZ(offsets), part_id); + optimiser.mydata.MDimg.getValue(EMDL_IMAGE_COORD_Z, zcoord, part_id); } else { @@ -223,7 +222,7 @@ class particle_reposition_parameters // Offsets in pixels offsets /= my_pixel_size; - optimiser.mydata.MDimg.getValue(EMDL_PARTICLE_CLASS, iclass, ori_img_id); + optimiser.mydata.MDimg.getValue(EMDL_PARTICLE_CLASS, iclass, part_id); iclass--; Euler_angles2matrix(rot, tilt, psi, A); @@ -253,7 +252,7 @@ class particle_reposition_parameters Image Ictf; FileName fn_ctf; - optimiser.mydata.MDimg.getValue(EMDL_CTF_IMAGE, fn_ctf, ori_img_id); + optimiser.mydata.MDimg.getValue(EMDL_CTF_IMAGE, fn_ctf, part_id); Ictf.read(fn_ctf); // If there is a redundant half, get rid of it @@ -279,7 +278,7 @@ class particle_reposition_parameters } else { - ctf.readByGroup(optimiser.mydata.MDimg, &optimiser.mydata.obsModel, ori_img_id); + ctf.readByGroup(optimiser.mydata.MDimg, &optimiser.mydata.obsModel, part_id); ctf.getFftwImage(Fctf, my_image_size, my_image_size, my_pixel_size, optimiser.ctf_phase_flipped, false, optimiser.intact_ctf_first_peak, true); } @@ -354,8 +353,8 @@ class particle_reposition_parameters RFLOAT part_avg, part_stdev; if (optimiser.do_helical_refine) { - optimiser.mydata.MDimg.getValue(EMDL_ORIENT_TILT_PRIOR, tilt_deg, ori_img_id); - optimiser.mydata.MDimg.getValue(EMDL_ORIENT_PSI_PRIOR, psi_deg, ori_img_id); + optimiser.mydata.MDimg.getValue(EMDL_ORIENT_TILT_PRIOR, tilt_deg, part_id); + optimiser.mydata.MDimg.getValue(EMDL_ORIENT_PSI_PRIOR, psi_deg, part_id); } calculateBackgroundAvgStddev(Ipart, part_avg, norm_factor, norm_radius, optimiser.do_helical_refine, @@ -365,7 +364,7 @@ class particle_reposition_parameters if (optimiser.do_norm_correction) { RFLOAT mynorm; - optimiser.mydata.MDimg.getValue(EMDL_IMAGE_NORM_CORRECTION, mynorm, ori_img_id); + optimiser.mydata.MDimg.getValue(EMDL_IMAGE_NORM_CORRECTION, mynorm, part_id); // TODO: check whether this is the right way around!!! norm_factor *= mynorm/optimiser.mymodel.avg_norm_correction; } diff --git a/src/class_ranker.cpp b/src/class_ranker.cpp index 40852c615..19d477947 100644 --- a/src/class_ranker.cpp +++ b/src/class_ranker.cpp @@ -549,7 +549,7 @@ void ClassRanker::initialise() if (!only_do_subimages && (intact_ctf_first_peak || do_ranking || (!do_skip_angular_errors && !haveAllAccuracies)) ) { // Read in particles (otherwise wait until haveAllAccuracies or performRanking, as Liyi sometimes doesn't need mydata) - mydata.read(fn_data, true, true); // true true means: ignore particle_name and group name! + mydata.read(fn_data, "", "", true, true); // true true means: ignore particle_name and group name! total_nr_particles = mydata.numberOfParticles(0); } @@ -1960,7 +1960,7 @@ void ClassRanker::performRanking() if (mydata.numberOfParticles() == 0) { // Read in particles if we haven't done this already - mydata.read(fn_data, true, true); // true true means: ignore particle_name and group name! + mydata.read(fn_data, "", "", true, true); // true true means: ignore particle_name and group name! if (debug>0) std::cerr << "Done with reading data.star ..." << std::endl; } diff --git a/src/exp_model.cpp b/src/exp_model.cpp index e0d45fd7b..72dd46da1 100644 --- a/src/exp_model.cpp +++ b/src/exp_model.cpp @@ -81,10 +81,6 @@ int Experiment::getRandomSubset(long int part_id) return particles[part_id].random_subset; } -int Experiment::getOriginalImageId(long part_id, int img_id) -{ - return particles[part_id].images[img_id].id; -} RFLOAT Experiment::getImagePixelSize(long int part_id, int img_id) { int optics_group = particles[part_id].images[img_id].optics_group; @@ -124,53 +120,44 @@ void Experiment::getNumberOfImagesPerOpticsGroup(std::vector &nr_parti } } -MetaDataTable Experiment::getMetaDataImage(long int part_id, int img_id) +MetaDataTable Experiment::getMetaDataParticle(long int part_id) { MetaDataTable result; - result.addObject(MDimg.getObject(getOriginalImageId(part_id, img_id))); + result.addObject(MDimg.getObject(part_id)); return result; } -FileName Experiment::getMicrographName(long int ori_image_id) +FileName Experiment::getMicrographName(long int part_id) { FileName micname=""; - if (is_3D) + if (is_3D || is_tomo) { - MDimg.getValue(EMDL_TOMO_NAME, micname, ori_image_id); + MDimg.getValue(EMDL_TOMO_NAME, micname, part_id); } else { - MDimg.getValue(EMDL_MICROGRAPH_NAME, micname, ori_image_id); + MDimg.getValue(EMDL_MICROGRAPH_NAME, micname, part_id); } - // SHWS 16112020: in relion-3.2 it is time to let go of the old polishing.... - // TODO: this is a temporary check: remove from distribution code!!! - if (micname.contains("@")) REPORT_ERROR("ERROR: micrographnames cannot have @ signs in them"); return micname; } -FileName Experiment::getMicrographName(long int part_id, int img_id) -{ - return getMicrographName(getOriginalImageId(part_id, img_id)); -} - -long int Experiment::addParticle(std::string part_name, int random_subset) +void Experiment::addParticle(int random_subset, int tomogram_id) { ExpParticle particle; - particle.name = part_name; + particle.id = particles.size(); + particle.tomogram_id = tomogram_id; particle.random_subset = random_subset; // Push back this particle in the particles vector and its sorted index in sorted_idx sorted_idx.push_back(particles.size()); particles.push_back(particle); - // Return the current part_id in the particles vector - return particles.size() - 1; + return; } -int Experiment::addImageToParticle(long int part_id, std::string img_name, long int ori_img_id, long int group_id, - int optics_group, bool unique) +void Experiment::addImageToParticle(std::string img_name, long int part_id, long int group_id, int optics_group) { if (group_id >= groups.size()) REPORT_ERROR("Experiment::addImageToParticle: group_id out of range"); @@ -180,12 +167,10 @@ int Experiment::addImageToParticle(long int part_id, std::string img_name, long ExpImage img; img.name = img_name; - img.id = ori_img_id; img.particle_id = part_id; img.group_id = group_id; img.optics_group = optics_group; - if (unique) - nr_images_per_optics_group[optics_group]++; + nr_images_per_optics_group[optics_group]++; img.optics_group_id = nr_images_per_optics_group[optics_group] - 1; if (img.optics_group_id < 0) @@ -194,7 +179,7 @@ int Experiment::addImageToParticle(long int part_id, std::string img_name, long // Push back this particle in the particles vector particles[part_id].images.push_back(img); - return particles[part_id].images.size() - 1; + return; } long int Experiment::addGroup(std::string group_name, int _optics_group) @@ -261,11 +246,10 @@ void Experiment::divideParticlesInRandomHalves(int seed, bool do_helical_refine) for (long int part_id = 0; part_id < particles.size(); part_id++) { // Get name of micrograph of the first image in this particle - mic_name = getMicrographName(part_id, 0); + mic_name = getMicrographName(part_id); if (divide_according_to_helical_tube_id) { - long int ori_img_id = getOriginalImageId(part_id, 0); - MDimg.getValue(EMDL_PARTICLE_HELICAL_TUBE_ID, helical_tube_id, ori_img_id); + MDimg.getValue(EMDL_PARTICLE_HELICAL_TUBE_ID, helical_tube_id, part_id); if (helical_tube_id < 1) REPORT_ERROR("ERROR Experiment::divideParticlesInRandomHalves: Helical tube ID should be positive integer!"); mic_name += std::string("_TUBEID_"); @@ -314,11 +298,10 @@ void Experiment::divideParticlesInRandomHalves(int seed, bool do_helical_refine) for (long int part_id = 0; part_id < particles.size(); part_id++) { // Get name of micrograph of the first image in this particle - mic_name = getMicrographName(part_id, 0); + mic_name = getMicrographName(part_id); if (divide_according_to_helical_tube_id) { - long int ori_img_id = getOriginalImageId(part_id, 0); - MDimg.getValue(EMDL_PARTICLE_HELICAL_TUBE_ID, helical_tube_id, ori_img_id); + MDimg.getValue(EMDL_PARTICLE_HELICAL_TUBE_ID, helical_tube_id, part_id); if (helical_tube_id < 1) REPORT_ERROR("ERROR Experiment::divideParticlesInRandomHalves: Helical tube ID should be positive integer!"); mic_name += std::string("_TUBEID_"); @@ -348,11 +331,7 @@ void Experiment::divideParticlesInRandomHalves(int seed, bool do_helical_refine) else REPORT_ERROR("ERROR Experiment::divideParticlesInRandomHalves: invalid number for random subset (i.e. not 1 or 2): " + integerToString(random_subset)); - for (int img_id = 0; img_id < numberOfImagesInParticle(part_id); img_id++) - { - long int ori_img_id = getOriginalImageId(part_id, img_id); - MDimg.setValue(EMDL_PARTICLE_RANDOM_SUBSET, random_subset, ori_img_id); - } + MDimg.setValue(EMDL_PARTICLE_RANDOM_SUBSET, random_subset, part_id); } } @@ -607,7 +586,10 @@ void Experiment::deleteDataOnScratch() void Experiment::copyParticlesToScratch(int verb, bool do_copy, bool also_do_ctf_image, RFLOAT keep_free_scratch_Gb) { - // This function relies on prepareScratchDirectory() being called before! + if (is_tomo) REPORT_ERROR("ERROR: copying particles to scratch has not yet been implemented for relion-4.1 2D stack STA...."); + // TODO: the loop over MDimg below should be replaced by a loop over all particles and img_id therein.... + + // This function relies on prepareScratchDirectory() being called before! long int nr_part = MDimg.numberOfObjects(); int barstep; @@ -765,7 +747,8 @@ void Experiment::copyParticlesToScratch(int verb, bool do_copy, bool also_do_ctf } // Read from file -void Experiment::read(FileName fn_exp, bool do_ignore_particle_name, bool do_ignore_group_name, bool do_preread_images, +void Experiment::read(FileName fn_exp, FileName fn_tomo, FileName fn_motion, + bool do_ignore_particle_name, bool do_ignore_group_name, bool do_preread_images, bool need_tiltpsipriors_for_helical_refine, int verb) { @@ -785,7 +768,9 @@ void Experiment::read(FileName fn_exp, bool do_ignore_particle_name, bool do_ign timer.tic(tread); #endif - // Only open stacks once and then read multiple images + is_tomo = false; + + // Only open stacks once and then read multiple images fImageHandler hFile; long int dump; FileName fn_stack, fn_open_stack=""; @@ -796,7 +781,7 @@ void Experiment::read(FileName fn_exp, bool do_ignore_particle_name, bool do_ign if (!fn_exp.isStarFile()) { - REPORT_ERROR("ERROR: relion-3.2 no longer accepts image stacks as input to refinement. Use a STAR file instead..."); + REPORT_ERROR("ERROR: Input particles should be provides in a STAR file."); } else { @@ -805,6 +790,16 @@ void Experiment::read(FileName fn_exp, bool do_ignore_particle_name, bool do_ign ObservationModel::loadSafely(fn_exp, obsModel, MDimg, "particles", verb); nr_images_per_optics_group.resize(obsModel.numberOfOpticsGroups(), 0); + if (fn_tomo != "") + { + // For now read in particle table twice: once into MDimg and once into particleSet... + // TODO: work with pointer to avoid duplication? + // TODO: ObservationModel::loadSafely(fn_exp, obsModel, MDimg, "particles", verb, true, true); + particleSet.read(fn_exp, fn_motion, verb>0); + tomogramSet.read(fn_tomo, verb>0); + is_tomo = true; + } + // Set is_3D from MDopt int mydim=2; obsModel.opticsMdt.getValue(EMDL_IMAGE_DIMENSIONALITY, mydim, 0); @@ -833,29 +828,15 @@ void Experiment::read(FileName fn_exp, bool do_ignore_particle_name, bool do_ign particles.reserve(MDimg.numberOfObjects()); // Now Loop over all objects in the metadata file and fill the logical tree of the experiment - long int first_part_id = -1; - FileName prev_mic_name = "/Unlikely$filename$?*!"; - FileName prev_img_name = "/Unlikely$filename$?*!"; - int prev_optics_group = -999; - //FOR_ALL_OBJECTS_IN_METADATA_TABLE(MDimg) - for (long int ori_img_id = 0; ori_img_id < MDimg.numberOfObjects(); ori_img_id++) + for (long int part_id = 0; part_id < MDimg.numberOfObjects(); part_id++) { // Get the optics group of this particle - int optics_group = obsModel.getOpticsGroup(MDimg, ori_img_id); + int optics_group = obsModel.getOpticsGroup(MDimg, part_id); #ifdef DEBUG_READ timer.tic(tgroup); #endif - FileName mic_name = getMicrographName(ori_img_id); - // Find last_part_id, which is the first part_id of this micrograph (for efficient searching of ori_particles below...) - if (mic_name != prev_mic_name) - { - prev_mic_name = mic_name; - first_part_id = particles.size(); - - } - // For example in particle_polishing the groups are not needed... if (!do_ignore_group_name) { @@ -863,11 +844,12 @@ void Experiment::read(FileName fn_exp, bool do_ignore_particle_name, bool do_ign // Check whether there is a group label, if not use a group for each micrograph if (MDimg.containsLabel(EMDL_MLMODEL_GROUP_NAME)) { - MDimg.getValue(EMDL_MLMODEL_GROUP_NAME, group_name, ori_img_id); + MDimg.getValue(EMDL_MLMODEL_GROUP_NAME, group_name, part_id); } else { - FileName fn_pre, fn_jobnr; + FileName mic_name = getMicrographName(part_id); + FileName fn_pre, fn_jobnr; decomposePipelineFileName(mic_name, fn_pre, fn_jobnr, group_name); } @@ -889,85 +871,76 @@ void Experiment::read(FileName fn_exp, bool do_ignore_particle_name, bool do_ign else { // All images belong to the same micrograph and group - group_id = addGroup("group", 0); + if (part_id == 0) group_id = addGroup("group", 0); + else group_id = 0; } + // The group number is only set upon reading: it is not read from the STAR file itself, + // there the only thing that matters is the order of the micrograph_names + // Write igroup+1, to start numbering at one instead of at zero + MDimg.setValue(EMDL_MLMODEL_GROUP_NO, group_id + 1, part_id); + #ifdef DEBUG_READ timer.toc(tgroup); #endif // If there is an EMDL_PARTICLE_RANDOM_SUBSET entry in the input STAR-file, then set the random_subset, otherwise use default (0) int my_random_subset; - if (!MDimg.getValue(EMDL_PARTICLE_RANDOM_SUBSET, my_random_subset, ori_img_id)) + if (!MDimg.getValue(EMDL_PARTICLE_RANDOM_SUBSET, my_random_subset, part_id)) { my_random_subset = 0; } - // Add this image to an existing particle, or create a new particle - std::string part_name; - long int part_id = -1; - - if (MDimg.containsLabel(EMDL_PARTICLE_NAME)) - MDimg.getValue(EMDL_PARTICLE_NAME, part_name, ori_img_id); - else - MDimg.getValue(EMDL_IMAGE_NAME, part_name, ori_img_id); - if (MDimg.containsLabel(EMDL_PARTICLE_NAME) && !do_ignore_particle_name) - { - // Only search ori_particles for the last (original) micrograph - for (long int i = first_part_id; i < particles.size(); i++) - { - if (particles[i].name == part_name) - { - part_id = i; - break; - } - } - } + FileName img_name; + MDimg.getValue(EMDL_IMAGE_NAME, img_name, part_id); - // If no particles with this name was found, - // or if no EMDL_PARTICLE_NAME in the input file, or if do_ignore_original_particle_name - // then add a new particle - if (part_id < 0) - { - part_id = addParticle(part_name, my_random_subset); - } + // Now get all the images for this particle (only one for SPA, multiple for STA) + if (is_tomo) + { - // Create a new image in this particle - FileName img_name; - MDimg.getValue(EMDL_IMAGE_NAME, img_name, ori_img_id); + // Find the tomogram this particle belongs to + std::string tomo_name = MDimg.getString(EMDL_TOMO_NAME, part_id); + int tomo_id = tomogramSet.getTomogramIndex(tomo_name); - bool do_cache = (prev_img_name != img_name || prev_optics_group != optics_group); -#ifdef DEBUG_SCRATCH - std::cerr << "prev_img_name = " << prev_img_name << " img_name = " << img_name << " prev_optics_group = " << prev_optics_group << " optics_group = " << optics_group << " do_cache = " << do_cache << std::endl; -#endif - prev_img_name = img_name; - prev_optics_group = optics_group; + // Add this particle to the Experiment, with its tomogram + addParticle(my_random_subset, tomo_id); - int img_id = addImageToParticle(part_id, img_name, ori_img_id, group_id, optics_group, do_cache); + // Add all images for this particle + const int fc = tomogramSet.getFrameCount(tomo_id); + for (int f = 0; f < fc; f++) + { + FileName my_name = integerToString(f) + "@" + img_name; + addImageToParticle(img_name, part_id, group_id, optics_group); + } - // The group number is only set upon reading: it is not read from the STAR file itself, - // there the only thing that matters is the order of the micrograph_names - // Write igroup+1, to start numbering at one instead of at zero - MDimg.setValue(EMDL_MLMODEL_GROUP_NO, group_id + 1, ori_img_id); + } + else + { + // Add this particle to the Experiment + addParticle(my_random_subset); + // Create a new image in this particle + addImageToParticle(img_name, part_id, group_id, optics_group); + } #ifdef DEBUG_READ - timer.tic(tori); + timer.tic(tori); #endif - - if (do_preread_images) - { - Image img; - img_name.decompose(dump, fn_stack); - if (fn_stack != fn_open_stack) - { - hFile.openFile(fn_stack, WRITE_READONLY); - fn_open_stack = fn_stack; - } - img.readFromOpenFile(img_name, hFile, -1, false); - img().setXmippOrigin(); - particles[part_id].images[img_id].img = img(); - } + if (do_preread_images) + { + Image img; + for (int img_id = 0; img_id < particles[part_id].numberOfImages(); img_id++) + { + particles[part_id].images[img_id].name.decompose(dump, fn_stack); + if (fn_stack != fn_open_stack) { + hFile.openFile(fn_stack, WRITE_READONLY); + fn_open_stack = fn_stack; + } + img.readFromOpenFile(particles[part_id].images[img_id].name, hFile, -1, false); + img().setXmippOrigin(); + particles[part_id].images[img_id].img = img(); + } + } #ifdef DEBUG_READ timer.toc(tori); @@ -976,7 +949,7 @@ void Experiment::read(FileName fn_exp, bool do_ignore_particle_name, bool do_ign #ifdef DEBUG_READ nr_read++; #endif - } // end loop over all objects in MDimg (ori_part_id) + } // end loop over all objects in MDimg (part_id) #ifdef DEBUG_READ timer.toc(tfill); @@ -1071,6 +1044,10 @@ void Experiment::read(FileName fn_exp, bool do_ignore_particle_name, bool do_ign } } + // Make sure the partTable in particleSet is kept the same as MDimg in Experiment, for reading from it in ml_optimiser.cpp + // TODO! Make use of pointers to avoid duplication of entire MDimg here... + if (is_tomo) particleSet.partTable = MDimg; + #ifdef DEBUG_READ timer.toc(tdef); @@ -1086,25 +1063,19 @@ void Experiment::read(FileName fn_exp, bool do_ignore_particle_name, bool do_ign // Write to file void Experiment::write(FileName fn_out) { - std::ofstream fh; - fh.open((fn_out).c_str(), std::ios::out); - if (!fh) - REPORT_ERROR( (std::string)"Experiment::write: Cannot write file: " + fn_out); + std::ofstream ofs(fn_out); - obsModel.opticsMdt.setName("optics"); - obsModel.opticsMdt.write(fh); + if (is_tomo) particleSet.genTable.write(ofs); - // Always write MDimg - MDimg.setName("particles"); - MDimg.write(fh); + obsModel.opticsMdt.write(ofs); + MDimg.write(ofs); - if (nr_bodies > 1) - { - for (int ibody = 0; ibody < nr_bodies; ibody++) - { - MDbodies[ibody].write(fh); - } - } + if (nr_bodies > 1) + { + for (int ibody = 0; ibody < nr_bodies; ibody++) + { + MDbodies[ibody].write(ofs); + } + } - fh.close(); } diff --git a/src/exp_model.h b/src/exp_model.h index 7d9f54110..eb110a1f3 100644 --- a/src/exp_model.h +++ b/src/exp_model.h @@ -27,6 +27,10 @@ #include "src/time.h" #include "src/ctf.h" #include +#include +#include +#include +#include /// Reserve large vectors with some reasonable estimate // Larger numbers will still be OK, but memory management might suffer @@ -37,8 +41,6 @@ class ExpImage { public: - // Position of the image in the original input STAR file - long int id; // To which particle does this image belong long int particle_id; @@ -47,7 +49,7 @@ class ExpImage long int optics_group_id; // Name of this image (by this name it will be recognised upon reading) - std::string name; + FileName name; // ID of the group that this image comes from long int group_id; @@ -67,7 +69,6 @@ class ExpImage // Copy constructor needed for work with vectors ExpImage(ExpImage const& copy) { - id = copy.id; particle_id = copy.particle_id; optics_group_id = copy.optics_group_id; name = copy.name; @@ -80,7 +81,7 @@ class ExpImage // Define assignment operator in terms of the copy constructor ExpImage& operator=(ExpImage const& copy) { - id = copy.id; + particle_id = copy.particle_id; optics_group_id = copy.optics_group_id; name = copy.name; @@ -95,10 +96,13 @@ class ExpParticle { public: - // Name of this particle (by this name all the images inside it will be grouped) - std::string name; + // Position of the image in the original input STAR file + long int id; - // Random subset this particle belongs to + // Which tomogram does this particle belong to + int tomogram_id; + + // Random subset this particle belongs to int random_subset; // Vector of all the images for this particle @@ -113,16 +117,18 @@ class ExpParticle // Copy constructor needed for work with vectors ExpParticle(ExpParticle const& copy) { - name = copy.name; - random_subset = copy.random_subset; + id = copy.id; + tomogram_id = copy.tomogram_id; + random_subset = copy.random_subset; images = copy.images; } // Define assignment operator in terms of the copy constructor ExpParticle& operator=(ExpParticle const& copy) { - name = copy.name; - random_subset = copy.random_subset; + id = copy.id; + tomogram_id = copy.tomogram_id; + random_subset = copy.random_subset; images = copy.images; return *this; } @@ -187,10 +193,17 @@ class Experiment // Number of images per optics group std::vector nr_images_per_optics_group; - // One large MetaDataTable for all images + // One large MetaDataTable for all particles MetaDataTable MDimg; - // Number of bodies in multi-body refinement + // TODO: Pointer to the relevant MetaDataTable for all particles (MDimg for SPA, particleSet.partTable for STA) + //MetaDataTable *ptrMDimg; + + // For subtomogram averaging in RELION-4.1.... + ParticleSet particleSet; + TomogramSet tomogramSet; + + // Number of bodies in multi-body refinement int nr_bodies; // Vector with MetaDataTables for orientations of different bodies in the multi-body refinement @@ -209,7 +222,7 @@ class Experiment RFLOAT free_space_Gb; // Is this sub-tomograms? - bool is_3D; + bool is_tomo, is_3D; // Empty Constructor Experiment() @@ -227,6 +240,7 @@ class Experiment groups.clear(); groups.reserve(MAX_NR_GROUPS); particles.clear(); // reserve upon reading + particleSet.clearParticles(); sorted_idx.clear(); nr_particles_subset1 = nr_particles_subset2 = 0; nr_bodies = 1; @@ -234,6 +248,7 @@ class Experiment nr_parts_on_scratch.clear(); free_space_Gb = 10; is_3D = false; + is_tomo = false; MDimg.clear(); MDimg.setIsList(false); MDbodies.clear(); @@ -270,9 +285,6 @@ class Experiment // Get the optics group to which the N'th image for this particle belongs int getOpticsGroup(long int part_id, int img_id); - // Get the original position in the input STAR file for the N'th image for this particle - int getOriginalImageId(long int part_id, int img_id); - // Get the pixel size for the N-th image of this particle RFLOAT getImagePixelSize(long int part_id, int img_id); @@ -283,18 +295,16 @@ class Experiment void getNumberOfImagesPerOpticsGroup(std::vector &nr_particles_per_group, int random_subset = 0); // Get the metadata-row for this image in a separate MetaDataTable - MetaDataTable getMetaDataImage(long int part_id, int img_id); + MetaDataTable getMetaDataParticle(long int part_id); // Which micrograph (or tomogram) doe this particle image comes from? - FileName getMicrographName(long int ori_img_id); - FileName getMicrographName(long int part_id, int img_id); + FileName getMicrographName(long int part_id); // Add a particle - long int addParticle(std::string part_name, int random_subset = 0); + void addParticle(int random_subset = 0, int tomogram_id = 0); // Add an image to the given particle - int addImageToParticle(long int part_id, std::string img_name, long int ori_img_id, long int group_id, - int optics_group, bool unique); + void addImageToParticle(std::string img_name, long int part_id, long int group_id, int optics_group); // Add a group long int addGroup(std::string mic_name, int optics_group); @@ -336,7 +346,7 @@ class Experiment // Read from file void read( - FileName fn_in, + FileName fn_in, FileName fn_tomo, FileName fn_motion, bool do_ignore_particle_name = false, bool do_ignore_group_name = false, bool do_preread_images = false, bool need_tiltpsipriors_for_helical_refine = false, int verb = 0); diff --git a/src/flex_analyser.cpp b/src/flex_analyser.cpp index 94c41cb96..4d50f7cf3 100644 --- a/src/flex_analyser.cpp +++ b/src/flex_analyser.cpp @@ -64,7 +64,7 @@ void FlexAnalyser::initialise() if (fn_data == "") REPORT_ERROR("ERROR: please provide the --data argument!"); else - data.read(fn_data); + data.read(fn_data, "", ""); if (verb > 0) std::cout << " Reading in model.star file ..." << std::endl; diff --git a/src/ml_optimiser.cpp b/src/ml_optimiser.cpp index 4e1a2e95e..0e89c6c47 100644 --- a/src/ml_optimiser.cpp +++ b/src/ml_optimiser.cpp @@ -553,7 +553,8 @@ void MlOptimiser::parseInitial(int argc, char **argv) // General optimiser I/O stuff int general_section = parser.addSection("General options"); fn_data = parser.getOption("--i", "Input particles (in a star-file)", ""); - fn_tomo = parser.getOption("--tomograms", "Star file with the tomograms (in a star-file)", ""); + fn_tomo = parser.getOption("--tomograms", "Star file with the tomograms", ""); + fn_motion = parser.getOption("--trajectories", "Star file with the tomogram motion trajectories", ""); fn_OS = parser.getOption("--ios", "Input tomo optimiser set file. It is used to set --i, --ref or --solvent_mask if they are not provided. Updated output optimiser set is created.", ""); fn_out = parser.getOption("--o", "Output rootname", ""); nr_iter = textToInteger(parser.getOption("--iter", "Maximum number of iterations to perform", "-1")); @@ -595,6 +596,11 @@ void MlOptimiser::parseInitial(int argc, char **argv) { if (!optimisationSet.getValue(EMDL_TOMO_TOMOGRAMS_FILE_NAME, fn_tomo)) REPORT_ERROR("No tomograms filename was found in file " + fn_OS); + } + if (fn_motion == "") + { + if (!optimisationSet.getValue(EMDL_TOMO_TRAJECTORIES_FILE_NAME, fn_motion)) + std::cout << " No motion trajectories were found in file " + fn_OS + ". Continuing without mask." << std::endl; } if (fn_ref == "None") { @@ -1130,6 +1136,8 @@ void MlOptimiser::read(FileName fn_in, int rank, bool do_prevent_preread) do_auto_sampling = false; if (!MD.getValue(EMDL_TOMO_TOMOGRAMS_FILE_NAME, fn_tomo)) fn_tomo = ""; + if (!MD.getValue(EMDL_TOMO_TRAJECTORIES_FILE_NAME, fn_motion)) + fn_motion = ""; // Initialise some stuff for first-iteration only (not relevant here...) do_calculate_initial_sigma_noise = false; @@ -1152,7 +1160,8 @@ void MlOptimiser::read(FileName fn_in, int rank, bool do_prevent_preread) bool do_preread = (do_preread_images) ? (do_parallel_disc_io || rank == 0) : false; if (do_prevent_preread) do_preread = false; bool is_helical_segment = (do_helical_refine) || ((mymodel.ref_dim == 2) && (helical_tube_outer_diameter > 0.)); - mydata.read(fn_data, false, false, do_preread, is_helical_segment); + + mydata.read(fn_data, fn_tomo, fn_motion, false, false, do_preread, is_helical_segment); #ifdef DEBUG_READ std::cerr<<"MlOptimiser::readStar before model."< 0.)); int myverb = (rank==0) ? 1 : 0; - mydata.read(fn_data, true, false, do_preread, is_helical_segment, myverb); // true means ignore original particle name + mydata.read(fn_data, fn_tomo, fn_motion, true, false, do_preread, is_helical_segment, myverb); // true means ignore original particle name // Without this check, the program crashes later. if (mydata.numberOfParticles() == 0) @@ -2474,6 +2485,10 @@ void MlOptimiser::calculateSumOfPowerSpectraAndAverageImage(MultidimArray img; @@ -8777,85 +8790,80 @@ void MlOptimiser::monitorHiddenVariableChanges(long int my_first_part_id, long i { long int part_id = mydata.sorted_idx[part_id_sorted]; - for (int img_id = 0; img_id < mydata.numberOfImagesInParticle(part_id); img_id++, metadata_offset++) - { - - long int ori_img_id = mydata.particles[part_id].images[img_id].id; - RFLOAT my_pixel_size = mydata.getImagePixelSize(part_id, img_id); - for (int ibody = 0; ibody < mymodel.nr_bodies; ibody++) - { + RFLOAT my_pixel_size = mydata.getImagePixelSize(part_id, 0); - if (mymodel.nr_bodies > 1 && mymodel.keep_fixed_bodies[ibody] > 0) - continue; + for (int ibody = 0; ibody < mymodel.nr_bodies; ibody++) + { - RFLOAT old_rot, old_tilt, old_psi, old_xoff, old_yoff, old_zoff = 0.; - RFLOAT rot, tilt, psi, xoff, yoff, zoff = 0.; - int old_iclass, iclass; + if (mymodel.nr_bodies > 1 && mymodel.keep_fixed_bodies[ibody] > 0) + continue; - if (mymodel.nr_bodies > 1) - { + RFLOAT old_rot, old_tilt, old_psi, old_xoff, old_yoff, old_zoff = 0.; + RFLOAT rot, tilt, psi, xoff, yoff, zoff = 0.; + int old_iclass, iclass; - // Old optimal parameters - mydata.MDbodies[ibody].getValue(EMDL_ORIENT_ROT, old_rot, ori_img_id); - mydata.MDbodies[ibody].getValue(EMDL_ORIENT_TILT, old_tilt, ori_img_id); - mydata.MDbodies[ibody].getValue(EMDL_ORIENT_PSI, old_psi, ori_img_id); - mydata.MDbodies[ibody].getValue(EMDL_ORIENT_ORIGIN_X_ANGSTROM, old_xoff, ori_img_id); - mydata.MDbodies[ibody].getValue(EMDL_ORIENT_ORIGIN_Y_ANGSTROM, old_yoff, ori_img_id); - if (mymodel.data_dim == 3) - { - mydata.MDbodies[ibody].getValue(EMDL_ORIENT_ORIGIN_Z_ANGSTROM, old_zoff, ori_img_id); - } - old_iclass = 0; - - // New optimal parameters - rot = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, 0 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS); - tilt = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, 1 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS); - psi = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, 2 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS); - xoff = my_pixel_size * DIRECT_A2D_ELEM(exp_metadata, metadata_offset, 3 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS); - yoff = my_pixel_size * DIRECT_A2D_ELEM(exp_metadata, metadata_offset, 4 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS); - if (mymodel.data_dim == 3) - zoff = my_pixel_size * DIRECT_A2D_ELEM(exp_metadata, metadata_offset, 5 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS); - iclass = 0; + if (mymodel.nr_bodies > 1) + { - } - else - { + // Old optimal parameters + mydata.MDbodies[ibody].getValue(EMDL_ORIENT_ROT, old_rot, part_id); + mydata.MDbodies[ibody].getValue(EMDL_ORIENT_TILT, old_tilt, part_id); + mydata.MDbodies[ibody].getValue(EMDL_ORIENT_PSI, old_psi, part_id); + mydata.MDbodies[ibody].getValue(EMDL_ORIENT_ORIGIN_X_ANGSTROM, old_xoff, part_id); + mydata.MDbodies[ibody].getValue(EMDL_ORIENT_ORIGIN_Y_ANGSTROM, old_yoff, part_id); + if (mymodel.data_dim == 3) + { + mydata.MDbodies[ibody].getValue(EMDL_ORIENT_ORIGIN_Z_ANGSTROM, old_zoff, part_id); + } + old_iclass = 0; + + // New optimal parameters + rot = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, 0 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS); + tilt = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, 1 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS); + psi = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, 2 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS); + xoff = my_pixel_size * DIRECT_A2D_ELEM(exp_metadata, metadata_offset, 3 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS); + yoff = my_pixel_size * DIRECT_A2D_ELEM(exp_metadata, metadata_offset, 4 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS); + if (mymodel.data_dim == 3) + zoff = my_pixel_size * DIRECT_A2D_ELEM(exp_metadata, metadata_offset, 5 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS); + iclass = 0; - // Old optimal parameters - mydata.MDimg.getValue(EMDL_ORIENT_ROT, old_rot, ori_img_id); - mydata.MDimg.getValue(EMDL_ORIENT_TILT, old_tilt, ori_img_id); - mydata.MDimg.getValue(EMDL_ORIENT_PSI, old_psi, ori_img_id); - mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_X_ANGSTROM, old_xoff, ori_img_id); - mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_Y_ANGSTROM, old_yoff, ori_img_id); - if (mymodel.data_dim == 3) - { - mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_Z_ANGSTROM, old_zoff, ori_img_id); - } - mydata.MDimg.getValue(EMDL_PARTICLE_CLASS, old_iclass, ori_img_id); - - // New optimal parameters - rot = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ROT); - tilt = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_TILT); - psi = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PSI); - xoff = my_pixel_size * DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_XOFF); - yoff = my_pixel_size * DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_YOFF); - if (mymodel.data_dim == 3) - zoff = my_pixel_size * DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ZOFF); - iclass = (int)DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_CLASS); + } + else + { - } + // Old optimal parameters + mydata.MDimg.getValue(EMDL_ORIENT_ROT, old_rot, part_id); + mydata.MDimg.getValue(EMDL_ORIENT_TILT, old_tilt, part_id); + mydata.MDimg.getValue(EMDL_ORIENT_PSI, old_psi, part_id); + mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_X_ANGSTROM, old_xoff, part_id); + mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_Y_ANGSTROM, old_yoff, part_id); + if (mymodel.data_dim == 3) + { + mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_Z_ANGSTROM, old_zoff, part_id); + } + mydata.MDimg.getValue(EMDL_PARTICLE_CLASS, old_iclass, part_id); + + // New optimal parameters + rot = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ROT); + tilt = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_TILT); + psi = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PSI); + xoff = my_pixel_size * DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_XOFF); + yoff = my_pixel_size * DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_YOFF); + if (mymodel.data_dim == 3) + zoff = my_pixel_size * DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ZOFF); + iclass = (int)DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_CLASS); - // Some orientational distance.... - sum_changes_optimal_orientations += sampling.calculateAngularDistance(rot, tilt, psi, old_rot, old_tilt, old_psi); - sum_changes_optimal_offsets += (xoff-old_xoff)*(xoff-old_xoff) + (yoff-old_yoff)*(yoff-old_yoff) + (zoff-old_zoff)*(zoff-old_zoff); - if (iclass != old_iclass) - sum_changes_optimal_classes += 1.; - sum_changes_count += 1.; + } - } // end loop ibody + // Some orientational distance.... + sum_changes_optimal_orientations += sampling.calculateAngularDistance(rot, tilt, psi, old_rot, old_tilt, old_psi); + sum_changes_optimal_offsets += (xoff-old_xoff)*(xoff-old_xoff) + (yoff-old_yoff)*(yoff-old_yoff) + (zoff-old_zoff)*(zoff-old_zoff); + if (iclass != old_iclass) + sum_changes_optimal_classes += 1.; + sum_changes_count += 1.; - } // end loop img_id + } // end loop ibody } //end loop part_id @@ -9105,8 +9113,8 @@ void MlOptimiser::calculateExpectedAngularErrors(long int my_first_part_id, long if ( (imode == 0 && ang_error > 30.) || (imode == 1 && sh_error > 10.) ) break; - // ori_img_id to keep exactly the same as in relion-3.0.... - init_random_generator(random_seed + mydata.getOriginalImageId(part_id, img_id)); + // part_id to keep exactly the same as in relion-3.0.... + init_random_generator(random_seed + part_id); MultidimArray F1, F2; Matrix2D A1, A2; @@ -9817,79 +9825,73 @@ void MlOptimiser::setMetaDataSubset(long int first_part_id, long int last_part_i { long int part_id = mydata.sorted_idx[part_id_sorted]; - for (int img_id = 0; img_id < mydata.numberOfImagesInParticle(part_id); img_id++, metadata_offset++) - { - - long int ori_img_id = mydata.particles[part_id].images[img_id].id; - RFLOAT my_pixel_size = mydata.getImagePixelSize(part_id, img_id); - - // SHWS: Upon request of Juha Huiskonen, 5apr2016 - if (mymodel.ref_dim > 2) - { - mydata.MDimg.setValue(EMDL_ORIENT_ROT, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ROT), ori_img_id); - mydata.MDimg.setValue(EMDL_ORIENT_TILT, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_TILT), ori_img_id); - } - mydata.MDimg.setValue(EMDL_ORIENT_PSI, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PSI), ori_img_id); - mydata.MDimg.setValue(EMDL_ORIENT_ORIGIN_X_ANGSTROM, my_pixel_size * DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_XOFF), ori_img_id); - mydata.MDimg.setValue(EMDL_ORIENT_ORIGIN_Y_ANGSTROM, my_pixel_size * DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_YOFF), ori_img_id); - if (mymodel.data_dim == 3) - { - mydata.MDimg.setValue(EMDL_ORIENT_ORIGIN_Z_ANGSTROM, my_pixel_size * DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ZOFF), ori_img_id); - } - mydata.MDimg.setValue(EMDL_PARTICLE_CLASS, (int)DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_CLASS) , ori_img_id); - mydata.MDimg.setValue(EMDL_PARTICLE_DLL, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_DLL), ori_img_id); - mydata.MDimg.setValue(EMDL_PARTICLE_PMAX, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PMAX), ori_img_id); - mydata.MDimg.setValue(EMDL_PARTICLE_NR_SIGNIFICANT_SAMPLES,(int)DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_NR_SIGN), ori_img_id); - mydata.MDimg.setValue(EMDL_IMAGE_NORM_CORRECTION, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_NORM), ori_img_id); - // For the moment, CTF, prior and transformation matrix info is NOT updated... - RFLOAT prior_x = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_XOFF_PRIOR); - RFLOAT prior_y = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_YOFF_PRIOR); - if (prior_x < 999.) - { - mydata.MDimg.setValue(EMDL_ORIENT_ORIGIN_X_PRIOR_ANGSTROM, my_pixel_size * prior_x, ori_img_id); - } - if (prior_y < 999.) - { - mydata.MDimg.setValue(EMDL_ORIENT_ORIGIN_Y_PRIOR_ANGSTROM, my_pixel_size * prior_y, ori_img_id); - } - if (mymodel.data_dim == 3) - { - RFLOAT prior_z = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ZOFF_PRIOR); - if (prior_z < 999.) - { - mydata.MDimg.setValue(EMDL_ORIENT_ORIGIN_Z_PRIOR_ANGSTROM, my_pixel_size * prior_z, ori_img_id); - } - } + RFLOAT my_pixel_size = mydata.getImagePixelSize(part_id, 0); - // For multi-body refinement - if (mymodel.nr_bodies > 1) - { - for (int ibody = 0; ibody < mymodel.nr_bodies; ibody++) - { - int icol_rot = 0 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; - int icol_tilt = 1 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; - int icol_psi = 2 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; - int icol_xoff = 3 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; - int icol_yoff = 4 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; - int icol_zoff = 5 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; - RFLOAT rot = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_rot); - RFLOAT tilt = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_tilt); - RFLOAT psi = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_psi); - RFLOAT xoff = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_xoff); - RFLOAT yoff = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_yoff); - RFLOAT zoff = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_zoff); - mydata.MDbodies[ibody].setValue(EMDL_ORIENT_ROT, rot, ori_img_id); - mydata.MDbodies[ibody].setValue(EMDL_ORIENT_TILT, tilt, ori_img_id); - mydata.MDbodies[ibody].setValue(EMDL_ORIENT_PSI, psi, ori_img_id); - mydata.MDbodies[ibody].setValue(EMDL_ORIENT_ORIGIN_X_ANGSTROM, my_pixel_size * xoff, ori_img_id); - mydata.MDbodies[ibody].setValue(EMDL_ORIENT_ORIGIN_Y_ANGSTROM, my_pixel_size * yoff, ori_img_id); - if (mymodel.data_dim == 3) - mydata.MDbodies[ibody].setValue(EMDL_ORIENT_ORIGIN_Z_ANGSTROM, my_pixel_size * zoff, ori_img_id); - } - } + if (mymodel.ref_dim > 2) + { + mydata.MDimg.setValue(EMDL_ORIENT_ROT, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ROT), part_id); + mydata.MDimg.setValue(EMDL_ORIENT_TILT, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_TILT), part_id); + } + mydata.MDimg.setValue(EMDL_ORIENT_PSI, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PSI), part_id); + mydata.MDimg.setValue(EMDL_ORIENT_ORIGIN_X_ANGSTROM, my_pixel_size * DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_XOFF), part_id); + mydata.MDimg.setValue(EMDL_ORIENT_ORIGIN_Y_ANGSTROM, my_pixel_size * DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_YOFF), part_id); + if (mymodel.data_dim == 3) + { + mydata.MDimg.setValue(EMDL_ORIENT_ORIGIN_Z_ANGSTROM, my_pixel_size * DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ZOFF), part_id); + } + mydata.MDimg.setValue(EMDL_PARTICLE_CLASS, (int)DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_CLASS) , part_id); + mydata.MDimg.setValue(EMDL_PARTICLE_DLL, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_DLL), part_id); + mydata.MDimg.setValue(EMDL_PARTICLE_PMAX, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PMAX), part_id); + mydata.MDimg.setValue(EMDL_PARTICLE_NR_SIGNIFICANT_SAMPLES,(int)DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_NR_SIGN), part_id); + mydata.MDimg.setValue(EMDL_IMAGE_NORM_CORRECTION, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_NORM), part_id); + + // For the moment, CTF, prior and transformation matrix info is NOT updated... + RFLOAT prior_x = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_XOFF_PRIOR); + RFLOAT prior_y = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_YOFF_PRIOR); + if (prior_x < 999.) + { + mydata.MDimg.setValue(EMDL_ORIENT_ORIGIN_X_PRIOR_ANGSTROM, my_pixel_size * prior_x, part_id); + } + if (prior_y < 999.) + { + mydata.MDimg.setValue(EMDL_ORIENT_ORIGIN_Y_PRIOR_ANGSTROM, my_pixel_size * prior_y, part_id); + } + if (mymodel.data_dim == 3) + { + RFLOAT prior_z = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ZOFF_PRIOR); + if (prior_z < 999.) + { + mydata.MDimg.setValue(EMDL_ORIENT_ORIGIN_Z_PRIOR_ANGSTROM, my_pixel_size * prior_z, part_id); + } + } - } // end for img_id + // For multi-body refinement + if (mymodel.nr_bodies > 1) + { + for (int ibody = 0; ibody < mymodel.nr_bodies; ibody++) + { + int icol_rot = 0 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; + int icol_tilt = 1 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; + int icol_psi = 2 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; + int icol_xoff = 3 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; + int icol_yoff = 4 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; + int icol_zoff = 5 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; + RFLOAT rot = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_rot); + RFLOAT tilt = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_tilt); + RFLOAT psi = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_psi); + RFLOAT xoff = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_xoff); + RFLOAT yoff = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_yoff); + RFLOAT zoff = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_zoff); + mydata.MDbodies[ibody].setValue(EMDL_ORIENT_ROT, rot, part_id); + mydata.MDbodies[ibody].setValue(EMDL_ORIENT_TILT, tilt, part_id); + mydata.MDbodies[ibody].setValue(EMDL_ORIENT_PSI, psi, part_id); + mydata.MDbodies[ibody].setValue(EMDL_ORIENT_ORIGIN_X_ANGSTROM, my_pixel_size * xoff, part_id); + mydata.MDbodies[ibody].setValue(EMDL_ORIENT_ORIGIN_Y_ANGSTROM, my_pixel_size * yoff, part_id); + if (mymodel.data_dim == 3) + mydata.MDbodies[ibody].setValue(EMDL_ORIENT_ORIGIN_Z_ANGSTROM, my_pixel_size * zoff, part_id); + } + } } // end for part_id @@ -9898,6 +9900,9 @@ void MlOptimiser::setMetaDataSubset(long int first_part_id, long int last_part_i void MlOptimiser::getMetaAndImageDataSubset(long int first_part_id, long int last_part_id, bool do_also_imagedata) { + // TODO!!! passing pre-read imagedata does not yet work for 2D stacks of tomo data.... + // Also logic of img_id needs checking below.... + // In case we're reading images here, only open stacks once and then read multiple images fImageHandler hFile; FileName fn_img, fn_stack, fn_open_stack="";; @@ -9961,27 +9966,26 @@ void MlOptimiser::getMetaAndImageDataSubset(long int first_part_id, long int las for (int img_id = 0; img_id < mydata.numberOfImagesInParticle(part_id); img_id++, metadata_offset++) { - long int ori_img_id = mydata.particles[part_id].images[img_id].id; RFLOAT my_pixel_size = mydata.getImagePixelSize(part_id, img_id); int my_image_size = mydata.getOpticsImageSize(mydata.getOpticsGroup(part_id, img_id)); // Get the image names from the MDimg table FileName fn_img="", fn_rec_img="", fn_ctf=""; if (!mydata.getImageNameOnScratch(part_id, img_id, fn_img)) - mydata.MDimg.getValue(EMDL_IMAGE_NAME, fn_img, ori_img_id); + mydata.MDimg.getValue(EMDL_IMAGE_NAME, fn_img, part_id); if (mymodel.data_dim == 3 && do_ctf_correction) { // Also read the CTF image from disc if (!mydata.getImageNameOnScratch(part_id, img_id, fn_ctf, true)) { - if (!mydata.MDimg.getValue(EMDL_CTF_IMAGE, fn_ctf, ori_img_id)) + if (!mydata.MDimg.getValue(EMDL_CTF_IMAGE, fn_ctf, part_id)) REPORT_ERROR("MlOptimiser::getMetaAndImageDataSubset ERROR: cannot find rlnCtfImage for 3D CTF correction!"); } } if (has_converged && do_use_reconstruct_images) { - mydata.MDimg.getValue(EMDL_IMAGE_RECONSTRUCT_NAME, fn_rec_img, ori_img_id); + mydata.MDimg.getValue(EMDL_IMAGE_RECONSTRUCT_NAME, fn_rec_img, part_id); } if (do_also_imagedata) @@ -10081,42 +10085,42 @@ void MlOptimiser::getMetaAndImageDataSubset(long int first_part_id, long int las // Now get the metadata int iaux; - mydata.MDimg.getValue(EMDL_ORIENT_ROT, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ROT), ori_img_id); - mydata.MDimg.getValue(EMDL_ORIENT_TILT, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_TILT), ori_img_id); - mydata.MDimg.getValue(EMDL_ORIENT_PSI, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PSI), ori_img_id); + mydata.MDimg.getValue(EMDL_ORIENT_ROT, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ROT), part_id); + mydata.MDimg.getValue(EMDL_ORIENT_TILT, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_TILT), part_id); + mydata.MDimg.getValue(EMDL_ORIENT_PSI, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PSI), part_id); RFLOAT xoff_A, yoff_A, zoff_A; - mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_X_ANGSTROM, xoff_A, ori_img_id); - mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_Y_ANGSTROM, yoff_A, ori_img_id); + mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_X_ANGSTROM, xoff_A, part_id); + mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_Y_ANGSTROM, yoff_A, part_id); DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_XOFF) = xoff_A / my_pixel_size; DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_YOFF) = yoff_A / my_pixel_size; if (mymodel.data_dim == 3) { - mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_Z_ANGSTROM, zoff_A, ori_img_id); + mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_Z_ANGSTROM, zoff_A, part_id); DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ZOFF) = zoff_A / my_pixel_size; } - mydata.MDimg.getValue(EMDL_PARTICLE_CLASS, iaux, ori_img_id); + mydata.MDimg.getValue(EMDL_PARTICLE_CLASS, iaux, part_id); DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_CLASS) = (RFLOAT)iaux; - mydata.MDimg.getValue(EMDL_PARTICLE_DLL, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_DLL), ori_img_id); - mydata.MDimg.getValue(EMDL_PARTICLE_PMAX, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PMAX), ori_img_id); + mydata.MDimg.getValue(EMDL_PARTICLE_DLL, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_DLL), part_id); + mydata.MDimg.getValue(EMDL_PARTICLE_PMAX, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PMAX), part_id); // 5jul17: we do not need EMDL_PARTICLE_NR_SIGNIFICANT_SAMPLES for calculations. Send randomsubset instead! if (do_split_random_halves) - mydata.MDimg.getValue(EMDL_PARTICLE_RANDOM_SUBSET, iaux, ori_img_id); + mydata.MDimg.getValue(EMDL_PARTICLE_RANDOM_SUBSET, iaux, part_id); else - mydata.MDimg.getValue(EMDL_PARTICLE_NR_SIGNIFICANT_SAMPLES, iaux, ori_img_id); + mydata.MDimg.getValue(EMDL_PARTICLE_NR_SIGNIFICANT_SAMPLES, iaux, part_id); DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_NR_SIGN) = (RFLOAT)iaux; - if (!mydata.MDimg.getValue(EMDL_IMAGE_NORM_CORRECTION, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_NORM), ori_img_id)) + if (!mydata.MDimg.getValue(EMDL_IMAGE_NORM_CORRECTION, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_NORM), part_id)) DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_NORM) = 1.; // If the priors are NOT set, then set their values to 999. - if (!mydata.MDimg.getValue(EMDL_ORIENT_ROT_PRIOR, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ROT_PRIOR), ori_img_id)) + if (!mydata.MDimg.getValue(EMDL_ORIENT_ROT_PRIOR, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ROT_PRIOR), part_id)) DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ROT_PRIOR) = 999.; - if (!mydata.MDimg.getValue(EMDL_ORIENT_TILT_PRIOR, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_TILT_PRIOR), ori_img_id)) + if (!mydata.MDimg.getValue(EMDL_ORIENT_TILT_PRIOR, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_TILT_PRIOR), part_id)) DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_TILT_PRIOR) = 999.; - if (!mydata.MDimg.getValue(EMDL_ORIENT_PSI_PRIOR, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PSI_PRIOR), ori_img_id)) + if (!mydata.MDimg.getValue(EMDL_ORIENT_PSI_PRIOR, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PSI_PRIOR), part_id)) DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PSI_PRIOR) = 999.; - if (mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_X_PRIOR_ANGSTROM, xoff_A, ori_img_id)) + if (mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_X_PRIOR_ANGSTROM, xoff_A, part_id)) { DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_XOFF_PRIOR) = xoff_A / my_pixel_size; } @@ -10124,7 +10128,7 @@ void MlOptimiser::getMetaAndImageDataSubset(long int first_part_id, long int las { DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_XOFF_PRIOR) = 999.; } - if (mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_Y_PRIOR_ANGSTROM, yoff_A, ori_img_id)) + if (mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_Y_PRIOR_ANGSTROM, yoff_A, part_id)) { DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_YOFF_PRIOR) = yoff_A / my_pixel_size; } @@ -10134,7 +10138,7 @@ void MlOptimiser::getMetaAndImageDataSubset(long int first_part_id, long int las } if (mymodel.data_dim == 3) { - if (mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_Z_PRIOR_ANGSTROM, zoff_A, ori_img_id)) + if (mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_Z_PRIOR_ANGSTROM, zoff_A, part_id)) { DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ZOFF_PRIOR) = zoff_A / my_pixel_size; } @@ -10143,7 +10147,7 @@ void MlOptimiser::getMetaAndImageDataSubset(long int first_part_id, long int las DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ZOFF_PRIOR) = 999.; } } - if (!mydata.MDimg.getValue(EMDL_ORIENT_PSI_PRIOR_FLIP_RATIO, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PSI_PRIOR_FLIP_RATIO), ori_img_id)) + if (!mydata.MDimg.getValue(EMDL_ORIENT_PSI_PRIOR_FLIP_RATIO, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PSI_PRIOR_FLIP_RATIO), part_id)) DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PSI_PRIOR_FLIP_RATIO) = 999.; // The following per-particle parameters are passed around through metadata @@ -10152,22 +10156,22 @@ void MlOptimiser::getMetaAndImageDataSubset(long int first_part_id, long int las { RFLOAT DeltafU, DeltafV, azimuthal_angle, Bfac, kfac, phase_shift; - if (!mydata.MDimg.getValue(EMDL_CTF_DEFOCUSU, DeltafU, ori_img_id)) + if (!mydata.MDimg.getValue(EMDL_CTF_DEFOCUSU, DeltafU, part_id)) DeltafU=0; - if (!mydata.MDimg.getValue(EMDL_CTF_DEFOCUSV, DeltafV, ori_img_id)) + if (!mydata.MDimg.getValue(EMDL_CTF_DEFOCUSV, DeltafV, part_id)) DeltafV=DeltafU; - if (!mydata.MDimg.getValue(EMDL_CTF_DEFOCUS_ANGLE, azimuthal_angle, ori_img_id)) + if (!mydata.MDimg.getValue(EMDL_CTF_DEFOCUS_ANGLE, azimuthal_angle, part_id)) azimuthal_angle=0; - if (!mydata.MDimg.getValue(EMDL_CTF_BFACTOR, Bfac, ori_img_id)) + if (!mydata.MDimg.getValue(EMDL_CTF_BFACTOR, Bfac, part_id)) Bfac=0.; - if (!mydata.MDimg.getValue(EMDL_CTF_SCALEFACTOR, kfac, ori_img_id)) + if (!mydata.MDimg.getValue(EMDL_CTF_SCALEFACTOR, kfac, part_id)) kfac=1.; - if (!mydata.MDimg.getValue(EMDL_CTF_PHASESHIFT, phase_shift, ori_img_id)) + if (!mydata.MDimg.getValue(EMDL_CTF_PHASESHIFT, phase_shift, part_id)) phase_shift=0.; DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_CTF_DEFOCUS_U) = DeltafU; @@ -10191,13 +10195,13 @@ void MlOptimiser::getMetaAndImageDataSubset(long int first_part_id, long int las int icol_yoff = 4 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; int icol_zoff = 5 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; RFLOAT rot, tilt, psi, xoff, yoff, zoff=0.; - mydata.MDbodies[ibody].getValue(EMDL_ORIENT_ROT, rot, ori_img_id); - mydata.MDbodies[ibody].getValue(EMDL_ORIENT_TILT, tilt, ori_img_id); - mydata.MDbodies[ibody].getValue(EMDL_ORIENT_PSI, psi, ori_img_id); - mydata.MDbodies[ibody].getValue(EMDL_ORIENT_ORIGIN_X_ANGSTROM, xoff, ori_img_id); - mydata.MDbodies[ibody].getValue(EMDL_ORIENT_ORIGIN_Y_ANGSTROM, yoff, ori_img_id); + mydata.MDbodies[ibody].getValue(EMDL_ORIENT_ROT, rot, part_id); + mydata.MDbodies[ibody].getValue(EMDL_ORIENT_TILT, tilt, part_id); + mydata.MDbodies[ibody].getValue(EMDL_ORIENT_PSI, psi, part_id); + mydata.MDbodies[ibody].getValue(EMDL_ORIENT_ORIGIN_X_ANGSTROM, xoff, part_id); + mydata.MDbodies[ibody].getValue(EMDL_ORIENT_ORIGIN_Y_ANGSTROM, yoff, part_id); if (mymodel.data_dim == 3) - mydata.MDbodies[ibody].getValue(EMDL_ORIENT_ORIGIN_Z_ANGSTROM, zoff, ori_img_id); + mydata.MDbodies[ibody].getValue(EMDL_ORIENT_ORIGIN_Z_ANGSTROM, zoff, part_id); DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_rot) = rot; DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_tilt) = tilt; DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_psi) = psi; diff --git a/src/ml_optimiser.h b/src/ml_optimiser.h index 1e384852d..e77e24189 100644 --- a/src/ml_optimiser.h +++ b/src/ml_optimiser.h @@ -40,10 +40,6 @@ #include "src/helix.h" #include "src/local_symmetry.h" #include "src/acc/settings.h" -#include -#include -#include -#include #include #define ML_SIGNIFICANT_WEIGHT 1.e-8 @@ -151,11 +147,8 @@ class MlOptimiser // Optimiser set table for subtomo MetaDataTable optimisationSet; - // For relion-4.1 tomogram-processing as 2D stacks - TomogramSet tomogramSet; - ParticleSet particleSet; - // FileName for tomogramSet - FileName fn_tomo; + // For relion-4.1 tomogram-processing as 2D stacks: filenames for tomogramSet and motionTrajectories + FileName fn_tomo, fn_motion; // Generate a 3D model from 2D particles de novo? bool is_3d_model; diff --git a/src/ml_optimiser_mpi.cpp b/src/ml_optimiser_mpi.cpp index e025fae5d..de6e014ff 100644 --- a/src/ml_optimiser_mpi.cpp +++ b/src/ml_optimiser_mpi.cpp @@ -433,7 +433,7 @@ void MlOptimiserMpi::initialise() } Experiment temp; - temp.read(fn_data); + temp.read(fn_data, "", "", true, true); int t_ori_size = temp.getOpticsImageSize(0); //temp.MDopt.getValue(EMDL_IMAGE_SIZE, t_ori_size, 0); diff --git a/src/particle_subtractor.cpp b/src/particle_subtractor.cpp index 086d9ca8c..63c1bdc14 100644 --- a/src/particle_subtractor.cpp +++ b/src/particle_subtractor.cpp @@ -104,7 +104,7 @@ void ParticleSubtractor::initialise(int _rank, int _size) { opt.mydata.clear(); bool is_helical_segment = (opt.do_helical_refine) || ((opt.mymodel.ref_dim == 2) && (opt.helical_tube_outer_diameter > 0.)); - opt.mydata.read(fn_sel, false, false, false, is_helical_segment); + opt.mydata.read(fn_sel, "", "", false, false, false, is_helical_segment); } divideLabour(rank, size, my_first_part_id, my_last_part_id); @@ -464,8 +464,7 @@ void ParticleSubtractor::subtractOneParticle(long int part_id, long int imgno, l { // Read the particle image Image img; - long int ori_img_id = opt.mydata.particles[part_id].images[imgno].id; - int optics_group = opt.mydata.getOpticsGroup(part_id, 0); + int optics_group = opt.mydata.getOpticsGroup(part_id, imgno); img.read(opt.mydata.particles[part_id].images[0].name); img().setXmippOrigin(); @@ -486,7 +485,7 @@ void ParticleSubtractor::subtractOneParticle(long int part_id, long int imgno, l int myclass = 0; if (!ignore_class && opt.mydata.MDimg.containsLabel(EMDL_PARTICLE_CLASS)) { - opt.mydata.MDimg.getValue(EMDL_PARTICLE_CLASS, myclass, ori_img_id); + opt.mydata.MDimg.getValue(EMDL_PARTICLE_CLASS, myclass, part_id); if (myclass > opt.mymodel.nr_classes) { std::cerr << "A particle belongs to class " << myclass << " while the number of classes in the optimiser.star is only " << opt.mymodel.nr_classes << "." << std::endl; @@ -494,17 +493,17 @@ void ParticleSubtractor::subtractOneParticle(long int part_id, long int imgno, l } myclass--; // start counting at zero instead of one } - opt.mydata.MDimg.getValue(EMDL_ORIENT_ROT, rot, ori_img_id); - opt.mydata.MDimg.getValue(EMDL_ORIENT_TILT, tilt, ori_img_id); - opt.mydata.MDimg.getValue(EMDL_ORIENT_PSI, psi, ori_img_id); - opt.mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_X_ANGSTROM, XX(my_old_offset), ori_img_id); - opt.mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_Y_ANGSTROM, YY(my_old_offset), ori_img_id); - if (opt.mymodel.data_dim == 3) opt.mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_Z_ANGSTROM, ZZ(my_old_offset), ori_img_id); + opt.mydata.MDimg.getValue(EMDL_ORIENT_ROT, rot, part_id); + opt.mydata.MDimg.getValue(EMDL_ORIENT_TILT, tilt, part_id); + opt.mydata.MDimg.getValue(EMDL_ORIENT_PSI, psi, part_id); + opt.mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_X_ANGSTROM, XX(my_old_offset), part_id); + opt.mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_Y_ANGSTROM, YY(my_old_offset), part_id); + if (opt.mymodel.data_dim == 3) opt.mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_Z_ANGSTROM, ZZ(my_old_offset), part_id); // As of v3.1, offsets are in Angstrom: convert back to pixels! my_old_offset /= my_pixel_size; // Apply the norm_correction term - if (!opt.mydata.MDimg.getValue(EMDL_IMAGE_NORM_CORRECTION, mynorm, ori_img_id)) mynorm = 1.; + if (!opt.mydata.MDimg.getValue(EMDL_IMAGE_NORM_CORRECTION, mynorm, part_id)) mynorm = 1.; if (opt.do_norm_correction) img() *= opt.mymodel.avg_norm_correction / mynorm; Matrix1D my_projected_com(3), my_refined_ibody_offset(3); @@ -544,7 +543,7 @@ void ParticleSubtractor::subtractOneParticle(long int part_id, long int imgno, l Image Ictf; FileName fn_ctf; - opt.mydata.MDimg.getValue(EMDL_CTF_IMAGE, fn_ctf, ori_img_id); + opt.mydata.MDimg.getValue(EMDL_CTF_IMAGE, fn_ctf, part_id); Ictf.read(fn_ctf); // If there is a redundant half, get rid of it @@ -596,8 +595,10 @@ void ParticleSubtractor::subtractOneParticle(long int part_id, long int imgno, l } else { - CTF ctf; - ctf.readByGroup(opt.mydata.MDimg, &opt.mydata.obsModel, ori_img_id); + // TODO! This will not yet work for 2D stacks in STA!!! + + CTF ctf; + ctf.readByGroup(opt.mydata.MDimg, &opt.mydata.obsModel, part_id); ctf.getFftwImage(Fctf, XSIZE(img()), YSIZE(img()), my_pixel_size, opt.ctf_phase_flipped, false, opt.intact_ctf_first_peak, true); @@ -624,14 +625,14 @@ void ParticleSubtractor::subtractOneParticle(long int part_id, long int imgno, l Matrix1D body_offset(3); RFLOAT body_rot, body_tilt, body_psi; - opt.mydata.MDbodies[obody].getValue(EMDL_ORIENT_ROT, body_rot, ori_img_id); - opt.mydata.MDbodies[obody].getValue(EMDL_ORIENT_TILT, body_tilt, ori_img_id); - opt.mydata.MDbodies[obody].getValue(EMDL_ORIENT_PSI, body_psi, ori_img_id); - opt.mydata.MDbodies[obody].getValue(EMDL_ORIENT_ORIGIN_X_ANGSTROM, XX(body_offset), ori_img_id); - opt.mydata.MDbodies[obody].getValue(EMDL_ORIENT_ORIGIN_Y_ANGSTROM, YY(body_offset), ori_img_id); + opt.mydata.MDbodies[obody].getValue(EMDL_ORIENT_ROT, body_rot, part_id); + opt.mydata.MDbodies[obody].getValue(EMDL_ORIENT_TILT, body_tilt, part_id); + opt.mydata.MDbodies[obody].getValue(EMDL_ORIENT_PSI, body_psi, part_id); + opt.mydata.MDbodies[obody].getValue(EMDL_ORIENT_ORIGIN_X_ANGSTROM, XX(body_offset), part_id); + opt.mydata.MDbodies[obody].getValue(EMDL_ORIENT_ORIGIN_Y_ANGSTROM, YY(body_offset), part_id); if (opt.mymodel.data_dim == 3) { - opt.mydata.MDbodies[obody].getValue(EMDL_ORIENT_ORIGIN_Z_ANGSTROM, ZZ(body_offset), ori_img_id); + opt.mydata.MDbodies[obody].getValue(EMDL_ORIENT_ORIGIN_Z_ANGSTROM, ZZ(body_offset), part_id); } // As of v3.1, offsets are in Angstrom: convert back to pixels! @@ -681,16 +682,16 @@ void ParticleSubtractor::subtractOneParticle(long int part_id, long int imgno, l Euler_matrix2angles(Abody, rot, tilt, psi); // Store the optimal orientations in the MDimg table - opt.mydata.MDimg.setValue(EMDL_ORIENT_ROT, rot, ori_img_id); - opt.mydata.MDimg.setValue(EMDL_ORIENT_TILT, tilt, ori_img_id); - opt.mydata.MDimg.setValue(EMDL_ORIENT_PSI, psi, ori_img_id); + opt.mydata.MDimg.setValue(EMDL_ORIENT_ROT, rot, part_id); + opt.mydata.MDimg.setValue(EMDL_ORIENT_TILT, tilt, part_id); + opt.mydata.MDimg.setValue(EMDL_ORIENT_PSI, psi, part_id); // Also get refined offset for this body - opt.mydata.MDbodies[subtract_body].getValue(EMDL_ORIENT_ORIGIN_X_ANGSTROM, XX(my_refined_ibody_offset), ori_img_id); - opt.mydata.MDbodies[subtract_body].getValue(EMDL_ORIENT_ORIGIN_Y_ANGSTROM, YY(my_refined_ibody_offset), ori_img_id); + opt.mydata.MDbodies[subtract_body].getValue(EMDL_ORIENT_ORIGIN_X_ANGSTROM, XX(my_refined_ibody_offset), part_id); + opt.mydata.MDbodies[subtract_body].getValue(EMDL_ORIENT_ORIGIN_Y_ANGSTROM, YY(my_refined_ibody_offset), part_id); if (opt.mymodel.data_dim == 3) { - opt.mydata.MDbodies[subtract_body].getValue(EMDL_ORIENT_ORIGIN_Z_ANGSTROM, ZZ(my_refined_ibody_offset), ori_img_id); + opt.mydata.MDbodies[subtract_body].getValue(EMDL_ORIENT_ORIGIN_Z_ANGSTROM, ZZ(my_refined_ibody_offset), part_id); } // As of v3.1, offsets are in Angstrom: convert back to pixels! my_refined_ibody_offset /= my_pixel_size; @@ -784,11 +785,11 @@ void ParticleSubtractor::subtractOneParticle(long int part_id, long int imgno, l selfTranslate(img(), centering_offset, WRAP); // Set the non-integer difference between the rounded centering offset and the actual offsets in the STAR file - opt.mydata.MDimg.setValue(EMDL_ORIENT_ORIGIN_X_ANGSTROM, my_pixel_size * XX(my_residual_offset), ori_img_id); - opt.mydata.MDimg.setValue(EMDL_ORIENT_ORIGIN_Y_ANGSTROM, my_pixel_size * YY(my_residual_offset), ori_img_id); + opt.mydata.MDimg.setValue(EMDL_ORIENT_ORIGIN_X_ANGSTROM, my_pixel_size * XX(my_residual_offset), part_id); + opt.mydata.MDimg.setValue(EMDL_ORIENT_ORIGIN_Y_ANGSTROM, my_pixel_size * YY(my_residual_offset), part_id); if (opt.mymodel.data_dim == 3) { - opt.mydata.MDimg.setValue(EMDL_ORIENT_ORIGIN_Z_ANGSTROM, my_pixel_size * ZZ(my_residual_offset), ori_img_id); + opt.mydata.MDimg.setValue(EMDL_ORIENT_ORIGIN_Z_ANGSTROM, my_pixel_size * ZZ(my_residual_offset), part_id); } } @@ -809,12 +810,12 @@ void ParticleSubtractor::subtractOneParticle(long int part_id, long int imgno, l // Now write out the image & set filenames in output metadatatable FileName fn_img = getParticleName(counter, rank, optics_group); - opt.mydata.MDimg.setValue(EMDL_IMAGE_NAME, fn_img, ori_img_id); - opt.mydata.MDimg.setValue(EMDL_IMAGE_ORI_NAME, opt.mydata.particles[part_id].images[0].name, ori_img_id); + opt.mydata.MDimg.setValue(EMDL_IMAGE_NAME, fn_img, part_id); + opt.mydata.MDimg.setValue(EMDL_IMAGE_ORI_NAME, opt.mydata.particles[part_id].images[0].name, part_id); //Also set the original order in the input STAR file for later combination - opt.mydata.MDimg.setValue(EMDL_IMAGE_ID, ori_img_id, ori_img_id); + opt.mydata.MDimg.setValue(EMDL_IMAGE_ID, part_id, part_id); MDimg_out.addObject(); - MDimg_out.setObject(opt.mydata.MDimg.getObject(ori_img_id)); + MDimg_out.setObject(opt.mydata.MDimg.getObject(part_id)); //printf("Writing: fn_orig = %s counter = %ld rank = %d optics_group = %d fn_img = %s SIZE = %d nr_particles_in_optics_group[optics_group] = %d\n", fn_orig.c_str(), counter, rank, optics_group+1, fn_img.c_str(), XSIZE(img()), nr_particles_in_optics_group[optics_group]); img.setSamplingRateInHeader(my_pixel_size); From 85695c5899732c594fcd48c4ef05ec26a0f7a4ed Mon Sep 17 00:00:00 2001 From: scheres Date: Tue, 5 Jul 2022 17:38:49 +0100 Subject: [PATCH 070/495] add projection angles for images in tomogram tilt series --- src/exp_model.cpp | 40 ++++++++++++++++++++++++++++++++++++++-- src/exp_model.h | 14 +++++++++++--- 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/src/exp_model.cpp b/src/exp_model.cpp index 72dd46da1..20faad582 100644 --- a/src/exp_model.cpp +++ b/src/exp_model.cpp @@ -19,6 +19,7 @@ ***************************************************************************/ #include "src/exp_model.h" #include +using namespace gravis; long int Experiment::numberOfParticles(int random_subset) { @@ -87,6 +88,11 @@ RFLOAT Experiment::getImagePixelSize(long int part_id, int img_id) return obsModel.getPixelSize(optics_group); } +Matrix2D Experiment::getRotationMatrix(long int part_id, int img_id) +{ + return particles[part_id].images[img_id].Aproj; +} + void Experiment::getNumberOfImagesPerGroup(std::vector &nr_particles_per_group, int random_subset) { nr_particles_per_group.resize(groups.size()); @@ -157,7 +163,7 @@ void Experiment::addParticle(int random_subset, int tomogram_id) return; } -void Experiment::addImageToParticle(std::string img_name, long int part_id, long int group_id, int optics_group) +void Experiment::addImageToParticle(std::string img_name, long int part_id, long int group_id, int optics_group, d4Matrix *Aproj) { if (group_id >= groups.size()) REPORT_ERROR("Experiment::addImageToParticle: group_id out of range"); @@ -165,11 +171,24 @@ void Experiment::addImageToParticle(std::string img_name, long int part_id, long if (optics_group >= obsModel.numberOfOpticsGroups()) REPORT_ERROR("Experiment::addImageToParticle: optics_group out of range"); + Matrix2D A(3,3); + if (Aproj == NULL) + { + A.initIdentity(); + } + else + { + for (int i = 0; i < 3; i++) + for (int j = 0; j < 3; j++) + A(i, j) = (*Aproj)(i, j); + } + ExpImage img; img.name = img_name; img.particle_id = part_id; img.group_id = group_id; img.optics_group = optics_group; + img.Aproj = A; nr_images_per_optics_group[optics_group]++; img.optics_group_id = nr_images_per_optics_group[optics_group] - 1; @@ -827,6 +846,9 @@ void Experiment::read(FileName fn_exp, FileName fn_tomo, FileName fn_motion, // allocate 1 block of memory particles.reserve(MDimg.numberOfObjects()); + Tomogram tomogram; + FileName prev_tomo_name = ""; + // Now Loop over all objects in the metadata file and fill the logical tree of the experiment for (long int part_id = 0; part_id < MDimg.numberOfObjects(); part_id++) { @@ -903,15 +925,29 @@ void Experiment::read(FileName fn_exp, FileName fn_tomo, FileName fn_motion, std::string tomo_name = MDimg.getString(EMDL_TOMO_NAME, part_id); int tomo_id = tomogramSet.getTomogramIndex(tomo_name); + if (tomo_name != prev_tomo_name) + { + tomogram = tomogramSet.loadTomogram(tomo_id, false); + prev_tomo_name = tomo_name; + } + // Add this particle to the Experiment, with its tomogram addParticle(my_random_subset, tomo_id); + // Pre-orientation of this particle in the tomogram + ParticleIndex id(part_id); + d3Matrix A = particleSet.getSubtomogramMatrix(id); + // Add all images for this particle const int fc = tomogramSet.getFrameCount(tomo_id); for (int f = 0; f < fc; f++) { + + d4Matrix P = tomogram.projectionMatrices[f] * d4Matrix(A); + FileName my_name = integerToString(f) + "@" + img_name; - addImageToParticle(img_name, part_id, group_id, optics_group); + + addImageToParticle(img_name, part_id, group_id, optics_group, &P); } } diff --git a/src/exp_model.h b/src/exp_model.h index eb110a1f3..d710bf755 100644 --- a/src/exp_model.h +++ b/src/exp_model.h @@ -35,6 +35,7 @@ /// Reserve large vectors with some reasonable estimate // Larger numbers will still be OK, but memory management might suffer #define MAX_NR_GROUPS 2000 +using namespace gravis; ////////////// Hierarchical metadata model @@ -60,7 +61,10 @@ class ExpImage // Pre-read array of the image in RAM MultidimArray img; - // Empty Constructor + // Projection matrix for tilt series stacks + Matrix2D Aproj; + + // Empty Constructor ExpImage() {} // Destructor needed for work with vectors @@ -75,7 +79,7 @@ class ExpImage group_id = copy.group_id; optics_group = copy.optics_group; img = copy.img; - + Aproj = copy.Aproj; } // Define assignment operator in terms of the copy constructor @@ -88,6 +92,7 @@ class ExpImage group_id = copy.group_id; optics_group = copy.optics_group; img = copy.img; + Aproj = copy.Aproj; return *this; } }; @@ -288,6 +293,9 @@ class Experiment // Get the pixel size for the N-th image of this particle RFLOAT getImagePixelSize(long int part_id, int img_id); + // Get the rotation matrix for Nth image of the subtomo particle + Matrix2D getRotationMatrix(long int part_id, int img_id); + // Get the vector of number of images per group_id void getNumberOfImagesPerGroup(std::vector &nr_particles_per_group, int random_subset = 0); @@ -304,7 +312,7 @@ class Experiment void addParticle(int random_subset = 0, int tomogram_id = 0); // Add an image to the given particle - void addImageToParticle(std::string img_name, long int part_id, long int group_id, int optics_group); + void addImageToParticle(std::string img_name, long int part_id, long int group_id, int optics_group, d4Matrix *Aproj = NULL); // Add a group long int addGroup(std::string mic_name, int optics_group); From c3e55c810bd8f8db20e26ea5fe7f0d3a4cbaadf1 Mon Sep 17 00:00:00 2001 From: scheres Date: Wed, 6 Jul 2022 11:03:26 +0100 Subject: [PATCH 071/495] add dz for defocus adjustment to is_tomo exp_model --- src/exp_model.cpp | 12 +++++++++--- src/exp_model.h | 7 ++++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/exp_model.cpp b/src/exp_model.cpp index 20faad582..d8feaee6c 100644 --- a/src/exp_model.cpp +++ b/src/exp_model.cpp @@ -163,7 +163,8 @@ void Experiment::addParticle(int random_subset, int tomogram_id) return; } -void Experiment::addImageToParticle(std::string img_name, long int part_id, long int group_id, int optics_group, d4Matrix *Aproj) +void Experiment::addImageToParticle(std::string img_name, long int part_id, long int group_id, int optics_group, + d4Matrix *Aproj, RFLOAT dz) { if (group_id >= groups.size()) REPORT_ERROR("Experiment::addImageToParticle: group_id out of range"); @@ -189,6 +190,7 @@ void Experiment::addImageToParticle(std::string img_name, long int part_id, long img.group_id = group_id; img.optics_group = optics_group; img.Aproj = A; + img.dz = dz; nr_images_per_optics_group[optics_group]++; img.optics_group_id = nr_images_per_optics_group[optics_group] - 1; @@ -937,17 +939,21 @@ void Experiment::read(FileName fn_exp, FileName fn_tomo, FileName fn_motion, // Pre-orientation of this particle in the tomogram ParticleIndex id(part_id); d3Matrix A = particleSet.getSubtomogramMatrix(id); + const d3Vector pos = particleSet.getParticleCoord(id); // Add all images for this particle const int fc = tomogramSet.getFrameCount(tomo_id); for (int f = 0; f < fc; f++) { + FileName my_name = integerToString(f) + "@" + img_name; + d4Matrix P = tomogram.projectionMatrices[f] * d4Matrix(A); - FileName my_name = integerToString(f) + "@" + img_name; + double dz_pos = tomogram.getDepthOffset(f, pos); + double dz = tomogram.handedness * tomogram.optics.pixelSize * tomogram.defocusSlope * dz_pos; - addImageToParticle(img_name, part_id, group_id, optics_group, &P); + addImageToParticle(img_name, part_id, group_id, optics_group, &P, dz); } } diff --git a/src/exp_model.h b/src/exp_model.h index d710bf755..b84229d95 100644 --- a/src/exp_model.h +++ b/src/exp_model.h @@ -64,6 +64,9 @@ class ExpImage // Projection matrix for tilt series stacks Matrix2D Aproj; + // Delta-Z for defocus adjustment of tilt seriers + RFLOAT dz; + // Empty Constructor ExpImage() {} @@ -80,6 +83,7 @@ class ExpImage optics_group = copy.optics_group; img = copy.img; Aproj = copy.Aproj; + dz = copy.dz; } // Define assignment operator in terms of the copy constructor @@ -93,6 +97,7 @@ class ExpImage optics_group = copy.optics_group; img = copy.img; Aproj = copy.Aproj; + dz = copy.dz; return *this; } }; @@ -312,7 +317,7 @@ class Experiment void addParticle(int random_subset = 0, int tomogram_id = 0); // Add an image to the given particle - void addImageToParticle(std::string img_name, long int part_id, long int group_id, int optics_group, d4Matrix *Aproj = NULL); + void addImageToParticle(std::string img_name, long int part_id, long int group_id, int optics_group, d4Matrix *Aproj = NULL, RFLOAT dz = 0.); // Add a group long int addGroup(std::string mic_name, int optics_group); From 67832b48deb08b0c3f3656363a818a2c544450ae Mon Sep 17 00:00:00 2001 From: scheres Date: Wed, 6 Jul 2022 12:24:44 +0100 Subject: [PATCH 072/495] went through getFTsandCTFs, exp_prior and exp_offset are no longer vectors --- src/acc/acc_ml_optimiser.h | 7 +- src/acc/acc_ml_optimiser_impl.h | 530 +++++++++++---------- src/ml_optimiser.cpp | 821 ++++++++++++++++---------------- src/ml_optimiser.h | 6 +- 4 files changed, 695 insertions(+), 669 deletions(-) diff --git a/src/acc/acc_ml_optimiser.h b/src/acc/acc_ml_optimiser.h index a82849574..cab6c6f31 100644 --- a/src/acc/acc_ml_optimiser.h +++ b/src/acc/acc_ml_optimiser.h @@ -132,7 +132,7 @@ class Indices class OptimisationParamters { public: - unsigned metadata_offset; + unsigned metadata_offset, imagedata_offset; unsigned long part_id; @@ -145,13 +145,14 @@ class OptimisationParamters // And from storeWeightedSums std::vector sum_weight, significant_weight, max_weight; std::vector< std::vector > sum_weight_class; - std::vector > old_offset, prior; + Matrix1D old_offset, prior; std::vector > power_img; MultidimArray Mweight; std::vector max_index; OptimisationParamters (unsigned nr_images, unsigned long part_id): metadata_offset(0), + imagedata_offset(0), part_id(part_id) { power_img.resize(nr_images); @@ -159,8 +160,6 @@ class OptimisationParamters Fimg.resize(nr_images); Fimg_nomask.resize(nr_images); Fctf.resize(nr_images); - old_offset.resize(nr_images); - prior.resize(nr_images); max_index.resize(nr_images); sum_weight_class.resize(nr_images); }; diff --git a/src/acc/acc_ml_optimiser_impl.h b/src/acc/acc_ml_optimiser_impl.h index c741c7ed0..40f924ed3 100644 --- a/src/acc/acc_ml_optimiser_impl.h +++ b/src/acc/acc_ml_optimiser_impl.h @@ -22,8 +22,213 @@ void getFourierTransformsAndCtfs(long int part_id, #endif CUSTOM_ALLOCATOR_REGION_NAME("GFTCTF"); + Matrix2D Aori; + Matrix1D my_projected_com(baseMLO->mymodel.data_dim), my_refined_ibody_offset(baseMLO->mymodel.data_dim); - for (int img_id = 0; img_id < sp.nr_images; img_id++) + // Get the norm_correction + RFLOAT normcorr = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_NORM); + + // Safeguard against gold-standard separation + if (baseMLO->do_split_random_halves) + { + int halfset = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_NR_SIGN); + if (halfset != baseMLO->my_halfset) + { + std::cerr << "BUG!!! halfset= " << halfset << " my_halfset= " << baseMLO->my_halfset << " part_id= " << part_id << std::endl; + REPORT_ERROR("BUG! Mixing gold-standard separation!!!!"); + } + + } + + // Get the optimal origin offsets from the previous iteration + // Sjors 5mar18: it is very important that my_old_offset has baseMLO->mymodel.data_dim and not just (3), as transformCartesianAndHelicalCoords will give different results!!! + Matrix1D my_old_offset(baseMLO->mymodel.data_dim), my_prior(baseMLO->mymodel.data_dim), my_old_offset_ori; + int icol_rot, icol_tilt, icol_psi, icol_xoff, icol_yoff, icol_zoff; + XX(my_old_offset) = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_XOFF); + YY(my_old_offset) = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_YOFF); + XX(my_prior) = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_XOFF_PRIOR); + YY(my_prior) = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_YOFF_PRIOR); + // Uninitialised priors were set to 999. + if (XX(my_prior) > 998.99 && XX(my_prior) < 999.01) + XX(my_prior) = 0.; + if (YY(my_prior) > 998.99 && YY(my_prior) < 999.01) + YY(my_prior) = 0.; + + if (accMLO->dataIs3D) + { + ZZ(my_old_offset) = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_ZOFF); + ZZ(my_prior) = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_ZOFF_PRIOR); + // Unitialised priors were set to 999. + if (ZZ(my_prior) > 998.99 && ZZ(my_prior) < 999.01) + ZZ(my_prior) = 0.; + } + + if (baseMLO->mymodel.nr_bodies > 1) + { + + // 17May2017: Shift image to the projected COM for this body! + // Aori is the original transformation matrix of the consensus refinement + Euler_angles2matrix(DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_ROT), + DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_TILT), + DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_PSI), Aori, false); + my_projected_com = Aori * baseMLO->mymodel.com_bodies[ibody]; + // This will have made my_projected_com of size 3 again! resize to mymodel.data_dim + my_projected_com.resize(baseMLO->mymodel.data_dim); + + // Subtract the projected COM offset, to position this body in the center + // Also keep the my_old_offset in my_old_offset_ori + my_old_offset_ori = my_old_offset; + my_old_offset -= my_projected_com; + + // Also get refined offset for this body + icol_xoff = 3 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; + icol_yoff = 4 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; + icol_zoff = 5 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; + XX(my_refined_ibody_offset) = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, icol_xoff); + YY(my_refined_ibody_offset) = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, icol_yoff); + if (baseMLO->mymodel.data_dim == 3) + ZZ(my_refined_ibody_offset) = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, icol_zoff); + + // For multi-body refinement: set the priors of the translations to zero (i.e. everything centred around consensus offset) + my_prior.initZeros(); + } + + // Store priors on translations + op.prior = my_prior; + + CTOC(accMLO->timer,"init"); + + CTIC(accMLO->timer,"nonZeroProb"); + // Orientational priors + if (baseMLO->mymodel.nr_bodies > 1 ) + { + + // Centre local searches around the orientation from the previous iteration, this one goes with overall sigma2_ang + // On top of that, apply prior on the deviation from (0,0,0) with mymodel.sigma_tilt_bodies[ibody] and mymodel.sigma_psi_bodies[ibody] + icol_rot = 0 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; + icol_tilt = 1 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; + icol_psi = 2 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; + RFLOAT prior_rot = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, icol_rot); + RFLOAT prior_tilt = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, icol_tilt); + RFLOAT prior_psi = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, icol_psi); + baseMLO->sampling.selectOrientationsWithNonZeroPriorProbability( + prior_rot, prior_tilt, prior_psi, + sqrt(baseMLO->mymodel.sigma2_rot), + sqrt(baseMLO->mymodel.sigma2_tilt), + sqrt(baseMLO->mymodel.sigma2_psi), + op.pointer_dir_nonzeroprior, op.directions_prior, + op.pointer_psi_nonzeroprior, op.psi_prior, false, 3., + baseMLO->mymodel.sigma_tilt_bodies[ibody], + baseMLO->mymodel.sigma_psi_bodies[ibody]); + + } + else if (baseMLO->mymodel.orientational_prior_mode != NOPRIOR && !(baseMLO->do_skip_align ||baseMLO-> do_skip_rotate)) + { + // First try if there are some fixed prior angles + // For multi-body refinements, ignore the original priors and get the refined residual angles from the previous iteration + RFLOAT prior_rot = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_ROT_PRIOR); + RFLOAT prior_tilt = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_TILT_PRIOR); + RFLOAT prior_psi = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_PSI_PRIOR); + RFLOAT prior_psi_flip_ratio = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_PSI_PRIOR_FLIP_RATIO); + + bool do_auto_refine_local_searches = (baseMLO->do_auto_refine) && (baseMLO->sampling.healpix_order >= baseMLO->autosampling_hporder_local_searches); + bool do_classification_local_searches = (! baseMLO->do_auto_refine) && (baseMLO->mymodel.orientational_prior_mode == PRIOR_ROTTILT_PSI) + && (baseMLO->mymodel.sigma2_rot > 0.) && (baseMLO->mymodel.sigma2_tilt > 0.) && (baseMLO->mymodel.sigma2_psi > 0.); + bool do_local_angular_searches = (do_auto_refine_local_searches) || (do_classification_local_searches); + + // If there were no defined priors (i.e. their values were 999.), then use the "normal" angles + if (prior_rot > 998.99 && prior_rot < 999.01) + prior_rot = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_ROT); + if (prior_tilt > 998.99 && prior_tilt < 999.01) + prior_tilt = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_TILT); + if (prior_psi > 998.99 && prior_psi < 999.01) + prior_psi = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_PSI); + if (prior_psi_flip_ratio > 998.99 && prior_psi_flip_ratio < 999.01) + prior_psi_flip_ratio = 0.5; + + ////////// How does this work now: each particle has a different sampling object?!!! + // Select only those orientations that have non-zero prior probability + + if (baseMLO->do_helical_refine && baseMLO->mymodel.ref_dim == 3) + { + baseMLO->sampling.selectOrientationsWithNonZeroPriorProbabilityFor3DHelicalReconstruction(prior_rot, prior_tilt, prior_psi, + sqrt(baseMLO->mymodel.sigma2_rot), sqrt(baseMLO->mymodel.sigma2_tilt), sqrt(baseMLO->mymodel.sigma2_psi), + op.pointer_dir_nonzeroprior, op.directions_prior, op.pointer_psi_nonzeroprior, op.psi_prior, + do_local_angular_searches, prior_psi_flip_ratio); + } + else + { + baseMLO->sampling.selectOrientationsWithNonZeroPriorProbability(prior_rot, prior_tilt, prior_psi, + sqrt(baseMLO->mymodel.sigma2_rot), sqrt(baseMLO->mymodel.sigma2_tilt), sqrt(baseMLO->mymodel.sigma2_psi), + op.pointer_dir_nonzeroprior, op.directions_prior, op.pointer_psi_nonzeroprior, op.psi_prior); + } + + long int nr_orients = baseMLO->sampling.NrDirections(0, &op.pointer_dir_nonzeroprior) * baseMLO->sampling.NrPsiSamplings(0, &op.pointer_psi_nonzeroprior); + if (nr_orients == 0) + { + std::cerr << " sampling.NrDirections()= " << baseMLO->sampling.NrDirections(0, &op.pointer_dir_nonzeroprior) + << " sampling.NrPsiSamplings()= " << baseMLO->sampling.NrPsiSamplings(0, &op.pointer_psi_nonzeroprior) << std::endl; + REPORT_ERROR("Zero orientations fall within the local angular search. Increase the sigma-value(s) on the orientations!"); + } + + } + CTOC(accMLO->timer,"nonZeroProb"); + + // Helical reconstruction: calculate old_offset in the system of coordinates of the helix, i.e. parallel & perpendicular, depending on psi-angle! + // For helices do NOT apply old_offset along the direction of the helix!! + Matrix1D my_old_offset_helix_coords; + RFLOAT rot_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_ROT); + RFLOAT tilt_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_TILT); + RFLOAT psi_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_PSI); + if ( (baseMLO->do_helical_refine) && (! baseMLO->ignore_helical_symmetry) ) + { + // Calculate my_old_offset_helix_coords from my_old_offset and psi angle + transformCartesianAndHelicalCoords(my_old_offset, my_old_offset_helix_coords, rot_deg, tilt_deg, psi_deg, CART_TO_HELICAL_COORDS); + // We do NOT want to accumulate the offsets in the direction along the helix (which is X in the helical coordinate system!) + // However, when doing helical local searches, we accumulate offsets + // Do NOT accumulate offsets in 3D classification of helices + if ( (! baseMLO->do_skip_align) && (! baseMLO->do_skip_rotate) ) + { + // TODO: check whether the following lines make sense + bool do_auto_refine_local_searches = (baseMLO->do_auto_refine) && (baseMLO->sampling.healpix_order >= baseMLO->autosampling_hporder_local_searches); + bool do_classification_local_searches = (! baseMLO->do_auto_refine) && (baseMLO->mymodel.orientational_prior_mode == PRIOR_ROTTILT_PSI) + && (baseMLO->mymodel.sigma2_rot > 0.) && (baseMLO->mymodel.sigma2_tilt > 0.) && (baseMLO->mymodel.sigma2_psi > 0.); + bool do_local_angular_searches = (do_auto_refine_local_searches) || (do_classification_local_searches); + if (!do_local_angular_searches) + { + if (! accMLO->dataIs3D) + XX(my_old_offset_helix_coords) = 0.; + else + ZZ(my_old_offset_helix_coords) = 0.; + } + } + // TODO: Now re-calculate the my_old_offset in the real (or image) system of coordinate (rotate -psi angle) + transformCartesianAndHelicalCoords(my_old_offset_helix_coords, my_old_offset, rot_deg, tilt_deg, psi_deg, HELICAL_TO_CART_COORDS); + } + CTOC(accMLO->timer,"HelicalPrep"); + + // ------------------------------------------------------------------------------------------ + + my_old_offset.selfROUND(); // Below, this rounded my_old_offset will be applied to the actual images + + // ------------------------------------------------------------------------------------------ + + if ( (baseMLO->do_helical_refine) && (! baseMLO->ignore_helical_symmetry) ) + { + // Transform rounded Cartesian offsets to corresponding helical ones + transformCartesianAndHelicalCoords(my_old_offset, my_old_offset_helix_coords, rot_deg, tilt_deg, psi_deg, CART_TO_HELICAL_COORDS); + op.old_offset = my_old_offset_helix_coords; + } + else + { + // For multi-bodies: store only the old refined offset, not the constant consensus offset or the projected COM of this body + if (baseMLO->mymodel.nr_bodies > 1) + op.old_offset = my_refined_ibody_offset; + else + op.old_offset = my_old_offset; // Not doing helical refinement. Rounded Cartesian offsets are stored. + } + + for (int img_id = 0; img_id < sp.nr_images; img_id++) { CTIC(accMLO->timer,"init"); FileName fn_img; @@ -31,8 +236,6 @@ void getFourierTransformsAndCtfs(long int part_id, MultidimArray Fimg; MultidimArray Faux; MultidimArray Fctf, FstMulti; - Matrix2D Aori; - Matrix1D my_projected_com(baseMLO->mymodel.data_dim), my_refined_ibody_offset(baseMLO->mymodel.data_dim); // Which group do I belong? int group_id =baseMLO->mydata.getGroupId(part_id, img_id); @@ -41,9 +244,6 @@ void getFourierTransformsAndCtfs(long int part_id, int optics_group = baseMLO->mydata.getOpticsGroup(part_id, img_id); bool ctf_premultiplied = baseMLO->mydata.obsModel.getCtfPremultiplied(optics_group); - // metadata offset for this image in the particle - int my_metadata_offset = op.metadata_offset + img_id; - // Get the right line in the exp_fn_img strings (also exp_fn_recimg and exp_fn_ctfs) int istop = 0; for (long int ii = baseMLO->exp_my_first_part_id; ii < part_id; ii++) @@ -53,160 +253,12 @@ void getFourierTransformsAndCtfs(long int part_id, if (!baseMLO->mydata.getImageNameOnScratch(part_id, img_id, fn_img)) { std::istringstream split(baseMLO->exp_fn_img); - for (int i = 0; i <= my_metadata_offset; i++) + for (int i = 0; i <= op.imagedata_offset; i++) getline(split, fn_img); } sp.current_img = fn_img; - // Get the norm_correction - RFLOAT normcorr = DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_NORM); - - // Safeguard against gold-standard separation - if (baseMLO->do_split_random_halves) - { - int halfset = DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_NR_SIGN); - if (halfset != baseMLO->my_halfset) - { - std::cerr << "BUG!!! halfset= " << halfset << " my_halfset= " << baseMLO->my_halfset << " part_id= " << part_id << std::endl; - REPORT_ERROR("BUG! Mixing gold-standard separation!!!!"); - } - - } - - // Get the optimal origin offsets from the previous iteration - // Sjors 5mar18: it is very important that my_old_offset has baseMLO->mymodel.data_dim and not just (3), as transformCartesianAndHelicalCoords will give different results!!! - Matrix1D my_old_offset(baseMLO->mymodel.data_dim), my_prior(baseMLO->mymodel.data_dim), my_old_offset_ori; - int icol_rot, icol_tilt, icol_psi, icol_xoff, icol_yoff, icol_zoff; - XX(my_old_offset) = DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_XOFF); - YY(my_old_offset) = DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_YOFF); - XX(my_prior) = DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_XOFF_PRIOR); - YY(my_prior) = DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_YOFF_PRIOR); - // Uninitialised priors were set to 999. - if (XX(my_prior) > 998.99 && XX(my_prior) < 999.01) - XX(my_prior) = 0.; - if (YY(my_prior) > 998.99 && YY(my_prior) < 999.01) - YY(my_prior) = 0.; - - if (accMLO->dataIs3D) - { - ZZ(my_old_offset) = DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_ZOFF); - ZZ(my_prior) = DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_ZOFF_PRIOR); - // Unitialised priors were set to 999. - if (ZZ(my_prior) > 998.99 && ZZ(my_prior) < 999.01) - ZZ(my_prior) = 0.; - } - - if (baseMLO->mymodel.nr_bodies > 1) - { - - // 17May2017: Shift image to the projected COM for this body! - // Aori is the original transformation matrix of the consensus refinement - Euler_angles2matrix(DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_ROT), - DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_TILT), - DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_PSI), Aori, false); - my_projected_com = Aori * baseMLO->mymodel.com_bodies[ibody]; - // This will have made my_projected_com of size 3 again! resize to mymodel.data_dim - my_projected_com.resize(baseMLO->mymodel.data_dim); - - // Subtract the projected COM offset, to position this body in the center - // Also keep the my_old_offset in my_old_offset_ori - my_old_offset_ori = my_old_offset; - my_old_offset -= my_projected_com; - - // Also get refined offset for this body - icol_xoff = 3 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; - icol_yoff = 4 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; - icol_zoff = 5 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; - XX(my_refined_ibody_offset) = DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, icol_xoff); - YY(my_refined_ibody_offset) = DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, icol_yoff); - if (baseMLO->mymodel.data_dim == 3) - ZZ(my_refined_ibody_offset) = DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, icol_zoff); - - // For multi-body refinement: set the priors of the translations to zero (i.e. everything centred around consensus offset) - my_prior.initZeros(); - } - - CTOC(accMLO->timer,"init"); - - CTIC(accMLO->timer,"nonZeroProb"); - // Orientational priors - if (baseMLO->mymodel.nr_bodies > 1 ) - { - - // Centre local searches around the orientation from the previous iteration, this one goes with overall sigma2_ang - // On top of that, apply prior on the deviation from (0,0,0) with mymodel.sigma_tilt_bodies[ibody] and mymodel.sigma_psi_bodies[ibody] - icol_rot = 0 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; - icol_tilt = 1 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; - icol_psi = 2 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; - RFLOAT prior_rot = DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, icol_rot); - RFLOAT prior_tilt = DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, icol_tilt); - RFLOAT prior_psi = DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, icol_psi); - baseMLO->sampling.selectOrientationsWithNonZeroPriorProbability( - prior_rot, prior_tilt, prior_psi, - sqrt(baseMLO->mymodel.sigma2_rot), - sqrt(baseMLO->mymodel.sigma2_tilt), - sqrt(baseMLO->mymodel.sigma2_psi), - op.pointer_dir_nonzeroprior, op.directions_prior, - op.pointer_psi_nonzeroprior, op.psi_prior, false, 3., - baseMLO->mymodel.sigma_tilt_bodies[ibody], - baseMLO->mymodel.sigma_psi_bodies[ibody]); - - } - else if (baseMLO->mymodel.orientational_prior_mode != NOPRIOR && !(baseMLO->do_skip_align ||baseMLO-> do_skip_rotate)) - { - // First try if there are some fixed prior angles - // For multi-body refinements, ignore the original priors and get the refined residual angles from the previous iteration - RFLOAT prior_rot = DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_ROT_PRIOR); - RFLOAT prior_tilt = DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_TILT_PRIOR); - RFLOAT prior_psi = DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_PSI_PRIOR); - RFLOAT prior_psi_flip_ratio = DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_PSI_PRIOR_FLIP_RATIO); - - bool do_auto_refine_local_searches = (baseMLO->do_auto_refine) && (baseMLO->sampling.healpix_order >= baseMLO->autosampling_hporder_local_searches); - bool do_classification_local_searches = (! baseMLO->do_auto_refine) && (baseMLO->mymodel.orientational_prior_mode == PRIOR_ROTTILT_PSI) - && (baseMLO->mymodel.sigma2_rot > 0.) && (baseMLO->mymodel.sigma2_tilt > 0.) && (baseMLO->mymodel.sigma2_psi > 0.); - bool do_local_angular_searches = (do_auto_refine_local_searches) || (do_classification_local_searches); - - // If there were no defined priors (i.e. their values were 999.), then use the "normal" angles - if (prior_rot > 998.99 && prior_rot < 999.01) - prior_rot = DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_ROT); - if (prior_tilt > 998.99 && prior_tilt < 999.01) - prior_tilt = DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_TILT); - if (prior_psi > 998.99 && prior_psi < 999.01) - prior_psi = DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_PSI); - if (prior_psi_flip_ratio > 998.99 && prior_psi_flip_ratio < 999.01) - prior_psi_flip_ratio = 0.5; - - ////////// How does this work now: each particle has a different sampling object?!!! - // Select only those orientations that have non-zero prior probability - - if (baseMLO->do_helical_refine && baseMLO->mymodel.ref_dim == 3) - { - baseMLO->sampling.selectOrientationsWithNonZeroPriorProbabilityFor3DHelicalReconstruction(prior_rot, prior_tilt, prior_psi, - sqrt(baseMLO->mymodel.sigma2_rot), sqrt(baseMLO->mymodel.sigma2_tilt), sqrt(baseMLO->mymodel.sigma2_psi), - op.pointer_dir_nonzeroprior, op.directions_prior, op.pointer_psi_nonzeroprior, op.psi_prior, - do_local_angular_searches, prior_psi_flip_ratio); - } - else - { - baseMLO->sampling.selectOrientationsWithNonZeroPriorProbability(prior_rot, prior_tilt, prior_psi, - sqrt(baseMLO->mymodel.sigma2_rot), sqrt(baseMLO->mymodel.sigma2_tilt), sqrt(baseMLO->mymodel.sigma2_psi), - op.pointer_dir_nonzeroprior, op.directions_prior, op.pointer_psi_nonzeroprior, op.psi_prior); - } - - long int nr_orients = baseMLO->sampling.NrDirections(0, &op.pointer_dir_nonzeroprior) * baseMLO->sampling.NrPsiSamplings(0, &op.pointer_psi_nonzeroprior); - if (nr_orients == 0) - { - std::cerr << " sampling.NrDirections()= " << baseMLO->sampling.NrDirections(0, &op.pointer_dir_nonzeroprior) - << " sampling.NrPsiSamplings()= " << baseMLO->sampling.NrPsiSamplings(0, &op.pointer_psi_nonzeroprior) << std::endl; - REPORT_ERROR("Zero orientations fall within the local angular search. Increase the sigma-value(s) on the orientations!"); - } - - } - CTOC(accMLO->timer,"nonZeroProb"); - - // ------------------------------------------------------------------------------------------ - - CTIC(accMLO->timer,"readData"); + CTIC(accMLO->timer,"readData"); // Get the image and recimg data if (baseMLO->do_parallel_disc_io) { @@ -235,7 +287,7 @@ void getFourierTransformsAndCtfs(long int part_id, else { CTIC(accMLO->timer,"ParaRead2DImages"); - img() = baseMLO->exp_imgs[my_metadata_offset]; + img() = baseMLO->exp_imgs[op.imagedata_offset]; CTOC(accMLO->timer,"ParaRead2DImages"); } } @@ -244,7 +296,7 @@ void getFourierTransformsAndCtfs(long int part_id, FileName fn_recimg; std::istringstream split2(baseMLO->exp_fn_recimg); // Get the right line in the exp_fn_img string - for (int i = 0; i <= my_metadata_offset; i++) + for (int i = 0; i <= op.imagedata_offset; i++) getline(split2, fn_recimg); rec_img.read(fn_recimg); rec_img().setXmippOrigin(); @@ -287,7 +339,7 @@ void getFourierTransformsAndCtfs(long int part_id, img().resize(baseMLO->image_full_size[optics_group], baseMLO->image_full_size[optics_group]); FOR_ALL_DIRECT_ELEMENTS_IN_ARRAY2D(img()) { - DIRECT_A2D_ELEM(img(), i, j) = DIRECT_A3D_ELEM(baseMLO->exp_imagedata, my_metadata_offset, i, j); + DIRECT_A2D_ELEM(img(), i, j) = DIRECT_A3D_ELEM(baseMLO->exp_imagedata, op.imagedata_offset, i, j); } img().setXmippOrigin(); if (baseMLO->has_converged && baseMLO->do_use_reconstruct_images) @@ -298,7 +350,7 @@ void getFourierTransformsAndCtfs(long int part_id, rec_img().resize(baseMLO->image_full_size[optics_group], baseMLO->image_full_size[optics_group]); FOR_ALL_DIRECT_ELEMENTS_IN_ARRAY2D(rec_img()) { - DIRECT_A2D_ELEM(rec_img(), i, j) = DIRECT_A3D_ELEM(baseMLO->exp_imagedata, my_nr_particles + my_metadata_offset, i, j); + DIRECT_A2D_ELEM(rec_img(), i, j) = DIRECT_A3D_ELEM(baseMLO->exp_imagedata, my_nr_particles + op.imagedata_offset, i, j); } rec_img().setXmippOrigin(); } @@ -367,46 +419,6 @@ void getFourierTransformsAndCtfs(long int part_id, * allocation-cost to that region. /BjoernF,160129 */ - // Apply (rounded) old offsets first - my_old_offset.selfROUND(); - - // Helical reconstruction: calculate old_offset in the system of coordinates of the helix, i.e. parallel & perpendicular, depending on psi-angle! - // For helices do NOT apply old_offset along the direction of the helix!! - Matrix1D my_old_offset_helix_coords; - RFLOAT rot_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_ROT); - RFLOAT tilt_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_TILT); - RFLOAT psi_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_PSI); - if ( (baseMLO->do_helical_refine) && (! baseMLO->ignore_helical_symmetry) ) - { - // Calculate my_old_offset_helix_coords from my_old_offset and psi angle - transformCartesianAndHelicalCoords(my_old_offset, my_old_offset_helix_coords, rot_deg, tilt_deg, psi_deg, CART_TO_HELICAL_COORDS); - // We do NOT want to accumulate the offsets in the direction along the helix (which is X in the helical coordinate system!) - // However, when doing helical local searches, we accumulate offsets - // Do NOT accumulate offsets in 3D classification of helices - if ( (! baseMLO->do_skip_align) && (! baseMLO->do_skip_rotate) ) - { - // TODO: check whether the following lines make sense - bool do_auto_refine_local_searches = (baseMLO->do_auto_refine) && (baseMLO->sampling.healpix_order >= baseMLO->autosampling_hporder_local_searches); - bool do_classification_local_searches = (! baseMLO->do_auto_refine) && (baseMLO->mymodel.orientational_prior_mode == PRIOR_ROTTILT_PSI) - && (baseMLO->mymodel.sigma2_rot > 0.) && (baseMLO->mymodel.sigma2_tilt > 0.) && (baseMLO->mymodel.sigma2_psi > 0.); - bool do_local_angular_searches = (do_auto_refine_local_searches) || (do_classification_local_searches); - if (!do_local_angular_searches) - { - if (! accMLO->dataIs3D) - XX(my_old_offset_helix_coords) = 0.; - else - ZZ(my_old_offset_helix_coords) = 0.; - } - } - // TODO: Now re-calculate the my_old_offset in the real (or image) system of coordinate (rotate -psi angle) - transformCartesianAndHelicalCoords(my_old_offset_helix_coords, my_old_offset, rot_deg, tilt_deg, psi_deg, HELICAL_TO_CART_COORDS); - } - CTOC(accMLO->timer,"HelicalPrep"); - - // ------------------------------------------------------------------------------------------ - - my_old_offset.selfROUND(); - // ------------------------------------------------------------------------------------------ CTIC(accMLO->timer,"TranslateAndNormCorrect"); @@ -417,6 +429,9 @@ void getFourierTransformsAndCtfs(long int part_id, d_img.allAlloc(); d_img.allInit(0); + // TODO!: think about applying 3D offsets as 2D shifts to the 2D STA stacks here... + if (baseMLO->mydata.is_tomo) REPORT_ERROR("GPU-version of translation for 2D stacks not yet implemented..."); + XFLOAT normcorr_val = baseMLO->do_norm_correction ? (XFLOAT)(baseMLO->mymodel.avg_norm_correction / normcorr) : 1; AccUtilities::TranslateAndNormCorrect( img.data, // input host-side MultidimArray d_img, // output acc-side Array @@ -475,25 +490,6 @@ void getFourierTransformsAndCtfs(long int part_id, // ------------------------------------------------------------------------------------------ - if ( (baseMLO->do_helical_refine) && (! baseMLO->ignore_helical_symmetry) ) - { - // Transform rounded Cartesian offsets to corresponding helical ones - transformCartesianAndHelicalCoords(my_old_offset, my_old_offset_helix_coords, rot_deg, tilt_deg, psi_deg, CART_TO_HELICAL_COORDS); - op.old_offset[img_id] = my_old_offset_helix_coords; - } - else - { - // For multi-bodies: store only the old refined offset, not the constant consensus offset or the projected COM of this body - if (baseMLO->mymodel.nr_bodies > 1) - op.old_offset[img_id] = my_refined_ibody_offset; - else - op.old_offset[img_id] = my_old_offset; // Not doing helical refinement. Rounded Cartesian offsets are stored. - } - // Also store priors on translations - op.prior[img_id] = my_prior; - - // ------------------------------------------------------------------------------------------ - CTIC(accMLO->timer,"selfApplyBeamTilt"); baseMLO->mydata.obsModel.demodulatePhase(optics_group, Fimg); baseMLO->mydata.obsModel.divideByMtf(optics_group, Fimg); @@ -525,6 +521,9 @@ void getFourierTransformsAndCtfs(long int part_id, d_img.streamSync(); d_img.getHost(img()); + // TODO! Think about how the direction of the individual images in tilt series stack changes the direction of the mask for STA!! + if (baseMLO->mydata.is_tomo) REPORT_ERROR("ERROR: softMaskOutsideMapForHelix for STA not implemented yet..."); + // ...modify img... if(baseMLO->do_zero_mask) { @@ -691,7 +690,7 @@ void getFourierTransformsAndCtfs(long int part_id, { std::istringstream split(baseMLO->exp_fn_ctf); // Get the right line in the exp_fn_img string - for (int i = 0; i <= my_metadata_offset; i++) + for (int i = 0; i <= op.imagedata_offset; i++) getline(split, fn_ctf); } Ictf.read(fn_ctf); @@ -719,12 +718,20 @@ void getFourierTransformsAndCtfs(long int part_id, CTF ctf; ctf.setValuesByGroup( &(baseMLO->mydata).obsModel, optics_group, - DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_CTF_DEFOCUS_U), - DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_CTF_DEFOCUS_V), - DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_CTF_DEFOCUS_ANGLE), - DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_CTF_BFACTOR), - DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_CTF_KFACTOR), - DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_CTF_PHASE_SHIFT)); + DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_CTF_DEFOCUS_U), + DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_CTF_DEFOCUS_V), + DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_CTF_DEFOCUS_ANGLE), + DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_CTF_BFACTOR), + DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_CTF_KFACTOR), + DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_CTF_PHASE_SHIFT)); + + // Apply the dz to the defocus for 2D image stacks in STA + if (baseMLO->mydata.is_tomo) + { + ctf.DeltafU += baseMLO->mydata.particles[part_id].images[img_id].dz; + ctf.DeltafV += baseMLO->mydata.particles[part_id].images[img_id].dz; + } + ctf.getFftwImage(Fctf, baseMLO->image_full_size[optics_group], baseMLO->image_full_size[optics_group], my_pixel_size, baseMLO->ctf_phase_flipped, baseMLO->only_flip_phases, baseMLO->intact_ctf_first_peak, true, baseMLO->do_ctf_padding); @@ -784,9 +791,9 @@ void getFourierTransformsAndCtfs(long int part_id, Matrix2D Aresi, Abody; // Aresi is the residual orientation for this obody - Euler_angles2matrix(DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, ocol_rot), - DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, ocol_tilt), - DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, ocol_psi), Aresi, false); + Euler_angles2matrix(DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, ocol_rot), + DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, ocol_tilt), + DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, ocol_psi), Aresi, false); // The real orientation to be applied is the obody transformation applied and the original one Abody = Aori * (baseMLO->mymodel.orient_bodies[obody]).transpose() * baseMLO->A_rot90 * Aresi * baseMLO->mymodel.orient_bodies[obody]; @@ -794,6 +801,9 @@ void getFourierTransformsAndCtfs(long int part_id, Abody = baseMLO->mydata.obsModel.applyAnisoMag(Abody, optics_group); Abody = baseMLO->mydata.obsModel.applyScaleDifference(Abody, optics_group, baseMLO->mymodel.ori_size, baseMLO->mymodel.pixel_size); + // TODO! The translation below will now work for 2D stacks in 2D, just like selfTranslate() didn't work upwards; Also need to apply per-image rotations, Aproj! + if (baseMLO->mydata.is_tomo) REPORT_ERROR("ERROR: multibody for STA not implemented yet!"); + // Get the FT of the projection in the right direction MultidimArray FTo; FTo.initZeros(Fimg); @@ -819,10 +829,10 @@ void getFourierTransformsAndCtfs(long int part_id, other_projected_com -= my_old_offset_ori; // Subtract refined obody-displacement - XX(other_projected_com) -= DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, ocol_xoff); - YY(other_projected_com) -= DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, ocol_yoff); + XX(other_projected_com) -= DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, ocol_xoff); + YY(other_projected_com) -= DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, ocol_yoff); if (baseMLO->mymodel.data_dim == 3) - ZZ(other_projected_com) -= DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, ocol_zoff); + ZZ(other_projected_com) -= DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, ocol_zoff); // Add the my_old_offset=selfRound(my_old_offset_ori - my_projected_com) already applied to this image for ibody other_projected_com += my_old_offset; @@ -944,7 +954,6 @@ void getAllSquaredDifferencesCoarse( if (baseMLO->mymodel.nr_bodies > 1) { - // img_id=0 because in multi-body refinement we do not do movie frames! RFLOAT rot_ori = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_ROT); RFLOAT tilt_ori = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_TILT); RFLOAT psi_ori = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_PSI); @@ -1014,7 +1023,6 @@ void getAllSquaredDifferencesCoarse( for (int img_id = 0; img_id < sp.nr_images; img_id++) { - int my_metadata_offset = op.metadata_offset + img_id; long int group_id = baseMLO->mydata.getGroupId(op.part_id, img_id); RFLOAT my_pixel_size = baseMLO->mydata.getImagePixelSize(op.part_id, img_id); int optics_group = baseMLO->mydata.getOpticsGroup(op.part_id, img_id); @@ -1058,9 +1066,9 @@ void getAllSquaredDifferencesCoarse( if ( (baseMLO->do_helical_refine) && (! baseMLO->ignore_helical_symmetry) ) { - RFLOAT rot_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_ROT); - RFLOAT tilt_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_TILT); - RFLOAT psi_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata,my_metadata_offset, METADATA_PSI); + RFLOAT rot_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_ROT); + RFLOAT tilt_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_TILT); + RFLOAT psi_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata,op.metadata_offset, METADATA_PSI); transformCartesianAndHelicalCoords(xshift, yshift, zshift, xshift, yshift, zshift, rot_deg, tilt_deg, psi_deg, (accMLO->dataIs3D) ? (3) : (2), HELICAL_TO_CART_COORDS); } @@ -1226,7 +1234,6 @@ void getAllSquaredDifferencesFine( // Reset size without de-allocating: we will append everything significant within // the current allocation and then re-allocate the then determined (smaller) volume - int my_metadata_offset = op.metadata_offset + img_id; long int group_id = baseMLO->mydata.getGroupId(op.part_id, img_id); RFLOAT my_pixel_size = baseMLO->mydata.getImagePixelSize(op.part_id, img_id); int optics_group = baseMLO->mydata.getOpticsGroup(op.part_id, img_id); @@ -1276,9 +1283,9 @@ void getAllSquaredDifferencesFine( if ( (baseMLO->do_helical_refine) && (! baseMLO->ignore_helical_symmetry) ) { - RFLOAT rot_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_ROT); - RFLOAT tilt_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_TILT); - RFLOAT psi_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_PSI); + RFLOAT rot_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_ROT); + RFLOAT tilt_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_TILT); + RFLOAT psi_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_PSI); transformCartesianAndHelicalCoords(xshift, yshift, zshift, xshift, yshift, zshift, rot_deg, tilt_deg, psi_deg, (accMLO->dataIs3D) ? (3) : (2), HELICAL_TO_CART_COORDS); } @@ -1628,7 +1635,6 @@ void convertAllSquaredDifferencesToWeights(unsigned exp_ipass, // loop over all images inside this particle for (int img_id = 0; img_id < sp.nr_images; img_id++) { - int my_metadata_offset = op.metadata_offset + img_id; RFLOAT my_pixel_size = baseMLO->mydata.getImagePixelSize(op.part_id, img_id); RFLOAT old_offset_x, old_offset_y, old_offset_z; @@ -1675,7 +1681,7 @@ void convertAllSquaredDifferencesToWeights(unsigned exp_ipass, #endif my_significant_weight = 0.999; - DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_NR_SIGN) = (RFLOAT) 1.; + DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_NR_SIGN) = (RFLOAT) 1.; if (exp_ipass==0) // TODO better memset, 0 => false , 1 => true for (int ihidden = 0; ihidden < XSIZE(op.Mcoarse_significant); ihidden++) if (DIRECT_A2D_ELEM(op.Mweight, img_id, ihidden) >= my_significant_weight) @@ -1733,9 +1739,9 @@ void convertAllSquaredDifferencesToWeights(unsigned exp_ipass, if (mypriors_len2 > 0.00001) { - RFLOAT rot_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_ROT); - RFLOAT tilt_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_TILT); - RFLOAT psi_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_PSI); + RFLOAT rot_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_ROT); + RFLOAT tilt_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_TILT); + RFLOAT psi_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_PSI); transformCartesianAndHelicalCoords(myprior_x, myprior_y, myprior_z, myprior_x, myprior_y, myprior_z, rot_deg, tilt_deg, psi_deg, (accMLO->dataIs3D) ? (3) : (2), CART_TO_HELICAL_COORDS); } } @@ -1916,7 +1922,7 @@ void convertAllSquaredDifferencesToWeights(unsigned exp_ipass, // Don't do this for multibody, as it would be overwritten for each body, // and we also use METADATA_NR_SIGN in the new safeguard for the gold-standard separation if (baseMLO->mymodel.nr_bodies == 1) - DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_NR_SIGN) = (RFLOAT) my_nr_significant_coarse_samples; + DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_NR_SIGN) = (RFLOAT) my_nr_significant_coarse_samples; AccPtr Mcoarse_significant = ptrFactory.make(ipart_length); Mcoarse_significant.setHostPtr(&op.Mcoarse_significant.data[offset]); @@ -2226,7 +2232,6 @@ void storeWeightedSums(OptimisationParamters &op, SamplingParameters &sp, oo_otrans.allAlloc(); int sumBlockNum =0; - int my_metadata_offset = op.metadata_offset + img_id; int group_id = baseMLO->mydata.getGroupId(op.part_id, img_id); const int optics_group = baseMLO->mydata.getOpticsGroup(op.part_id, img_id); RFLOAT my_pixel_size = baseMLO->mydata.getImagePixelSize(op.part_id, img_id); @@ -2294,9 +2299,9 @@ void storeWeightedSums(OptimisationParamters &op, SamplingParameters &sp, // If it is doing helical refinement AND Cartesian vector myprior has a length > 0, transform the vector to its helical coordinates if ( (baseMLO->do_helical_refine) && (! baseMLO->ignore_helical_symmetry) && (mypriors_len2 > 0.00001) ) { - RFLOAT rot_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_ROT); - RFLOAT tilt_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_TILT); - RFLOAT psi_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_PSI); + RFLOAT rot_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_ROT); + RFLOAT tilt_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_TILT); + RFLOAT psi_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_PSI); transformCartesianAndHelicalCoords(myprior_x, myprior_y, myprior_z, myprior_x, myprior_y, myprior_z, rot_deg, tilt_deg, psi_deg, (accMLO->dataIs3D) ? (3) : (2), CART_TO_HELICAL_COORDS); } @@ -2423,7 +2428,6 @@ void storeWeightedSums(OptimisationParamters &op, SamplingParameters &sp, std::vector< RFLOAT> oversampled_rot, oversampled_tilt, oversampled_psi; for (long int img_id = 0; img_id < sp.nr_images; img_id++) { - int my_metadata_offset = op.metadata_offset + img_id; RFLOAT my_pixel_size = baseMLO->mydata.getImagePixelSize(op.part_id, img_id); CTIC(accMLO->timer,"setMetadata"); @@ -2459,12 +2463,12 @@ void storeWeightedSums(OptimisationParamters &op, SamplingParameters &sp, int icol_yoff = (baseMLO->mymodel.nr_bodies == 1) ? METADATA_YOFF : 4 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; int icol_zoff = (baseMLO->mymodel.nr_bodies == 1) ? METADATA_ZOFF : 5 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; - RFLOAT old_rot = DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, icol_rot); - DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, icol_rot) = rot; - RFLOAT old_tilt = DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, icol_tilt); - DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, icol_tilt) = tilt; - RFLOAT old_psi = DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, icol_psi); - DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, icol_psi) = psi; + RFLOAT old_rot = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, icol_rot); + DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, icol_rot) = rot; + RFLOAT old_tilt = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, icol_tilt); + DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, icol_tilt) = tilt; + RFLOAT old_psi = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, icol_psi); + DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, icol_psi) = psi; Matrix1D shifts(baseMLO->mymodel.data_dim); @@ -2479,18 +2483,18 @@ void storeWeightedSums(OptimisationParamters &op, SamplingParameters &sp, if ( (baseMLO->do_helical_refine) && (! baseMLO->ignore_helical_symmetry) ) transformCartesianAndHelicalCoords(shifts, shifts, old_rot, old_tilt, old_psi, HELICAL_TO_CART_COORDS); - DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, icol_xoff) = XX(shifts); - DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, icol_yoff) = YY(shifts); + DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, icol_xoff) = XX(shifts); + DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, icol_yoff) = YY(shifts); if (accMLO->dataIs3D) - DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, icol_zoff) = ZZ(shifts); + DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, icol_zoff) = ZZ(shifts); if (ibody == 0) { - DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_CLASS) = (RFLOAT)op.max_index[img_id].iclass + 1; + DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_CLASS) = (RFLOAT)op.max_index[img_id].iclass + 1; RFLOAT pmax = op.max_weight[img_id]/op.sum_weight[img_id]; if(pmax>1) //maximum normalised probability weight is (unreasonably) larger than unity CRITICAL("Relion is finding a normalised probability greater than 1"); - DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_PMAX) = pmax; + DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_PMAX) = pmax; } CTOC(accMLO->timer,"setMetadata"); } diff --git a/src/ml_optimiser.cpp b/src/ml_optimiser.cpp index 0e89c6c47..5093fb23c 100644 --- a/src/ml_optimiser.cpp +++ b/src/ml_optimiser.cpp @@ -4131,11 +4131,13 @@ void MlOptimiser::expectationOneParticle(long int part_id_sorted, int thread_id) int my_nr_images = mydata.numberOfImagesInParticle(part_id); // Global exp_metadata array has metadata of all ori_particles. Where does my_ori_particle start? int metadata_offset = 0; - for (long int iori = exp_my_first_part_id; iori <= exp_my_last_part_id; iori++) + int imagedata_offset = 0; + for (long int iori = exp_my_first_part_id; iori <= exp_my_last_part_id; iori++) { if (iori == part_id_sorted) break; - metadata_offset += mydata.numberOfImagesInParticle(mydata.sorted_idx[iori]); + imagedata_offset += mydata.numberOfImagesInParticle(mydata.sorted_idx[iori]); + metadata_offset += 1; } // Resize vectors for all particles @@ -4160,7 +4162,7 @@ void MlOptimiser::expectationOneParticle(long int part_id_sorted, int thread_id) timer.tic(TIMING_ESP_FT); } #endif - getFourierTransformsAndCtfs(part_id, ibody, metadata_offset, exp_Fimg, exp_Fimg_nomask, exp_Fctf, + getFourierTransformsAndCtfs(part_id, ibody, metadata_offset, imagedata_offset, exp_Fimg, exp_Fimg_nomask, exp_Fctf, exp_old_offset, exp_prior, exp_power_imgs, exp_highres_Xi2_img, exp_pointer_dir_nonzeroprior, exp_pointer_psi_nonzeroprior, exp_directions_prior, exp_psi_prior, exp_STMulti); @@ -5485,12 +5487,12 @@ void MlOptimiser::updateImageSizeAndResolutionPointers() void MlOptimiser::getFourierTransformsAndCtfs( - long int part_id, int ibody, int metadata_offset, + long int part_id, int ibody, int metadata_offset, int imagedata_offset, std::vector > &exp_Fimg, std::vector > &exp_Fimg_nomask, std::vector > &exp_Fctf, - std::vector > &exp_old_offset, - std::vector > &exp_prior, + Matrix1D &exp_old_offset, + Matrix1D &exp_prior, std::vector > &exp_power_img, std::vector &exp_highres_Xi2_img, std::vector &exp_pointer_dir_nonzeroprior, @@ -5500,184 +5502,295 @@ void MlOptimiser::getFourierTransformsAndCtfs( std::vector > &exp_STMulti) { - FourierTransformer transformer; - for (int img_id = 0; img_id < mydata.numberOfImagesInParticle(part_id); img_id++) - { - Image img, rec_img; - MultidimArray Fimg, Faux; - MultidimArray Fctf, FstMulti; // SubtomoWeights - Matrix2D Aori; - Matrix1D my_projected_com(mymodel.data_dim), my_refined_ibody_offset(mymodel.data_dim); - - // To which group do I belong? - int group_id = mydata.getGroupId(part_id, img_id); - // What is my optics group? - int optics_group = mydata.getOpticsGroup(part_id, img_id); - bool ctf_premultiplied = mydata.obsModel.getCtfPremultiplied(optics_group); - RFLOAT my_pixel_size = mydata.getOpticsPixelSize(optics_group); - int my_image_size = mydata.getOpticsImageSize(optics_group); - - // metadata offset for this image in the particle - int my_metadata_offset = metadata_offset + img_id; + Matrix2D Aori; + Matrix1D my_projected_com(mymodel.data_dim), my_refined_ibody_offset(mymodel.data_dim); - // Get the norm_correction (for multi-body refinement: still use the one from the consensus refinement!) - RFLOAT normcorr = DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_NORM); + // Get the norm_correction (for multi-body refinement: still use the one from the consensus refinement!) + RFLOAT normcorr = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_NORM); - // Safeguard against gold-standard separation - if (do_split_random_halves) - { - int halfset = DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_NR_SIGN); - if (halfset != my_halfset) - { - std::cerr << "BUG!!! halfset= " << halfset << " my_halfset= " << my_halfset << " part_id= " << part_id << std::endl; - REPORT_ERROR("BUG! Mixing gold-standard separation!!!!"); - } - } + // Safeguard against gold-standard separation + if (do_split_random_halves) + { + int halfset = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_NR_SIGN); + if (halfset != my_halfset) + { + std::cerr << "BUG!!! halfset= " << halfset << " my_halfset= " << my_halfset << " part_id= " << part_id << std::endl; + REPORT_ERROR("BUG! Mixing gold-standard separation!!!!"); + } + } - // Get the old offsets and the priors on the offsets - // Sjors 5mar18: it is very important that my_old_offset has baseMLO->mymodel.data_dim and not just (3), as transformCartesianAndHelicalCoords will give different results!!! - Matrix1D my_old_offset(mymodel.data_dim), my_prior(mymodel.data_dim), my_old_offset_ori; + // Get the old offsets and the priors on the offsets + // Sjors 5mar18: it is very important that my_old_offset has baseMLO->mymodel.data_dim and not just (3), as transformCartesianAndHelicalCoords will give different results!!! + Matrix1D my_old_offset(mymodel.data_dim), my_prior(mymodel.data_dim), my_old_offset_ori; - int icol_rot, icol_tilt, icol_psi, icol_xoff, icol_yoff, icol_zoff; - XX(my_old_offset) = DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_XOFF); - XX(my_prior) = DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_XOFF_PRIOR); - YY(my_old_offset) = DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_YOFF); - YY(my_prior) = DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_YOFF_PRIOR); - if (mymodel.data_dim == 3) - { - ZZ(my_old_offset) = DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_ZOFF); - ZZ(my_prior) = DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_ZOFF_PRIOR); - } - if (mymodel.nr_bodies > 1) - { + int icol_rot, icol_tilt, icol_psi, icol_xoff, icol_yoff, icol_zoff; + XX(my_old_offset) = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_XOFF); + XX(my_prior) = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_XOFF_PRIOR); + YY(my_old_offset) = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_YOFF); + YY(my_prior) = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_YOFF_PRIOR); + if (mymodel.data_dim == 3) + { + ZZ(my_old_offset) = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ZOFF); + ZZ(my_prior) = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ZOFF_PRIOR); + } + if (mymodel.nr_bodies > 1) + { - // 17May2017: Shift image to the projected COM for this body! - // Aori is the original transformation matrix of the consensus refinement - Euler_angles2matrix(DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_ROT), - DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_TILT), - DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_PSI), Aori, false); - my_projected_com = Aori * mymodel.com_bodies[ibody]; - // This will have made my_projected_com of size 3 again! resize to mymodel.data_dim - my_projected_com.resize(mymodel.data_dim); + // 17May2017: Shift image to the projected COM for this body! + // Aori is the original transformation matrix of the consensus refinement + Euler_angles2matrix(DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ROT), + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_TILT), + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PSI), Aori, false); + my_projected_com = Aori * mymodel.com_bodies[ibody]; + // This will have made my_projected_com of size 3 again! resize to mymodel.data_dim + my_projected_com.resize(mymodel.data_dim); #ifdef DEBUG_BODIES - if (part_id == ROUND(debug1)) + if (part_id == ROUND(debug1)) { std::cerr << "ibody: " << ibody+1 << " projected COM: " << XX(my_projected_com) << " , " << YY(my_projected_com) << std::endl; std::cerr << "ibody: " << ibody+1 << " consensus offset: " << XX(my_old_offset) << " , " << YY(my_old_offset) << std::endl; } #endif - // Subtract the projected COM offset, to position this body in the center - // Also keep the my_old_offset in my_old_offset_ori - my_old_offset_ori = my_old_offset; - my_old_offset -= my_projected_com; + // Subtract the projected COM offset, to position this body in the center + // Also keep the my_old_offset in my_old_offset_ori + my_old_offset_ori = my_old_offset; + my_old_offset -= my_projected_com; - // Also get refined offset for this body - icol_xoff = 3 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; - icol_yoff = 4 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; - icol_zoff = 5 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; - XX(my_refined_ibody_offset) = DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, icol_xoff); - YY(my_refined_ibody_offset) = DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, icol_yoff); - if (mymodel.data_dim == 3) - ZZ(my_refined_ibody_offset) = DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, icol_zoff); + // Also get refined offset for this body + icol_xoff = 3 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; + icol_yoff = 4 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; + icol_zoff = 5 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; + XX(my_refined_ibody_offset) = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_xoff); + YY(my_refined_ibody_offset) = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_yoff); + if (mymodel.data_dim == 3) + ZZ(my_refined_ibody_offset) = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_zoff); - // For multi-body refinement: set the priors of the translations to zero (i.e. everything centred around consensus offset) - my_prior.initZeros(); + // For multi-body refinement: set the priors of the translations to zero (i.e. everything centred around consensus offset) + my_prior.initZeros(); #ifdef DEBUG_BODIES - if (part_id == ROUND(debug1)) + if (part_id == ROUND(debug1)) { - std::cerr << "ibody: " << ibody+1 << " refined x,y= " << DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, icol_xoff) - << " , " << DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, icol_yoff) << std::endl; + std::cerr << "ibody: " << ibody+1 << " refined x,y= " << DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_xoff) + << " , " << DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_yoff) << std::endl; std::cerr << "FINAL translation ibody: " << ibody+1 << " : " << XX(my_old_offset) << " , " << YY(my_old_offset) << std::endl; } #endif - } + } - // Uninitialised priors were set to 999. - if (XX(my_prior) > 998.99 && XX(my_prior) < 999.01) - XX(my_prior) = 0.; - if (YY(my_prior) > 998.99 && YY(my_prior) < 999.01) - YY(my_prior) = 0.; - if (mymodel.data_dim == 3 && ZZ(my_prior) > 998.99 && ZZ(my_prior) < 999.01) - ZZ(my_prior) = 0.; + // Uninitialised priors were set to 999. + if (XX(my_prior) > 998.99 && XX(my_prior) < 999.01) + XX(my_prior) = 0.; + if (YY(my_prior) > 998.99 && YY(my_prior) < 999.01) + YY(my_prior) = 0.; + if (mymodel.data_dim == 3 && ZZ(my_prior) > 998.99 && ZZ(my_prior) < 999.01) + ZZ(my_prior) = 0.; - // Orientational priors - if (mymodel.nr_bodies > 1 ) - { + // Store the prior on translations + exp_prior = my_prior; - // Centre local searches around the orientation from the previous iteration, this one goes with overall sigma2_ang - // On top of that, apply prior on the deviation from (0,0,0) with mymodel.sigma_tilt_bodies[ibody] and mymodel.sigma_psi_bodies[ibody] - icol_rot = 0 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; - icol_tilt = 1 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; - icol_psi = 2 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; - RFLOAT prior_rot = DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, icol_rot); - RFLOAT prior_tilt = DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, icol_tilt); - RFLOAT prior_psi = DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, icol_psi); - sampling.selectOrientationsWithNonZeroPriorProbability(prior_rot, prior_tilt, prior_psi, - sqrt(mymodel.sigma2_rot), sqrt(mymodel.sigma2_tilt), sqrt(mymodel.sigma2_psi), - exp_pointer_dir_nonzeroprior, exp_directions_prior, - exp_pointer_psi_nonzeroprior, exp_psi_prior, false, 3., - mymodel.sigma_tilt_bodies[ibody], mymodel.sigma_psi_bodies[ibody]); + // Orientational priors + if (mymodel.nr_bodies > 1 ) + { - } - else if (mymodel.orientational_prior_mode != NOPRIOR && !(do_skip_align || do_skip_rotate || do_only_sample_tilt)) - { - // First try if there are some fixed prior angles - // For multi-body refinements, ignore the original priors and get the refined residual angles from the previous iteration - RFLOAT prior_rot = DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_ROT_PRIOR); - RFLOAT prior_tilt = DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_TILT_PRIOR); - RFLOAT prior_psi = DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_PSI_PRIOR); - RFLOAT prior_psi_flip_ratio = DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_PSI_PRIOR_FLIP_RATIO); - RFLOAT prior_rot_flip_ratio = DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_ROT_PRIOR_FLIP_RATIO); // Kthurber + // Centre local searches around the orientation from the previous iteration, this one goes with overall sigma2_ang + // On top of that, apply prior on the deviation from (0,0,0) with mymodel.sigma_tilt_bodies[ibody] and mymodel.sigma_psi_bodies[ibody] + icol_rot = 0 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; + icol_tilt = 1 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; + icol_psi = 2 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; + RFLOAT prior_rot = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_rot); + RFLOAT prior_tilt = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_tilt); + RFLOAT prior_psi = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_psi); + sampling.selectOrientationsWithNonZeroPriorProbability(prior_rot, prior_tilt, prior_psi, + sqrt(mymodel.sigma2_rot), sqrt(mymodel.sigma2_tilt), sqrt(mymodel.sigma2_psi), + exp_pointer_dir_nonzeroprior, exp_directions_prior, + exp_pointer_psi_nonzeroprior, exp_psi_prior, false, 3., + mymodel.sigma_tilt_bodies[ibody], mymodel.sigma_psi_bodies[ibody]); - bool do_auto_refine_local_searches = (do_auto_refine || do_auto_sampling) && (sampling.healpix_order >= autosampling_hporder_local_searches); - bool do_classification_local_searches = (!do_auto_refine) && (mymodel.orientational_prior_mode == PRIOR_ROTTILT_PSI) - && (mymodel.sigma2_rot > 0.) && (mymodel.sigma2_tilt > 0.) && (mymodel.sigma2_psi > 0.); - bool do_local_angular_searches = (do_auto_refine_local_searches) || (do_classification_local_searches); + } + else if (mymodel.orientational_prior_mode != NOPRIOR && !(do_skip_align || do_skip_rotate || do_only_sample_tilt)) + { + // First try if there are some fixed prior angles + // For multi-body refinements, ignore the original priors and get the refined residual angles from the previous iteration + RFLOAT prior_rot = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ROT_PRIOR); + RFLOAT prior_tilt = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_TILT_PRIOR); + RFLOAT prior_psi = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PSI_PRIOR); + RFLOAT prior_psi_flip_ratio = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PSI_PRIOR_FLIP_RATIO); + RFLOAT prior_rot_flip_ratio = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ROT_PRIOR_FLIP_RATIO); // Kthurber + + bool do_auto_refine_local_searches = (do_auto_refine || do_auto_sampling) && (sampling.healpix_order >= autosampling_hporder_local_searches); + bool do_classification_local_searches = (!do_auto_refine) && (mymodel.orientational_prior_mode == PRIOR_ROTTILT_PSI) + && (mymodel.sigma2_rot > 0.) && (mymodel.sigma2_tilt > 0.) && (mymodel.sigma2_psi > 0.); + bool do_local_angular_searches = (do_auto_refine_local_searches) || (do_classification_local_searches); + + // If there were no defined priors (i.e. their values were 999.), then use the "normal" angles + if (prior_rot > 998.99 && prior_rot < 999.01) + prior_rot = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ROT); + if (prior_tilt > 998.99 && prior_tilt < 999.01) + prior_tilt = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_TILT); + if (prior_psi > 998.99 && prior_psi < 999.01) + prior_psi = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PSI); + if (prior_psi_flip_ratio > 998.99 && prior_psi_flip_ratio < 999.01) + prior_psi_flip_ratio = 0.5; + if (prior_rot_flip_ratio > 998.99 && prior_rot_flip_ratio < 999.01) // Kthurber + prior_rot_flip_ratio = 0.5; // Kthurber + + // Select only those orientations that have non-zero prior probability + // Jun04,2015 - Shaoda & Sjors, bimodal psi searches for helices + if (do_helical_refine && mymodel.ref_dim == 3) + { + sampling.selectOrientationsWithNonZeroPriorProbabilityFor3DHelicalReconstruction(prior_rot, prior_tilt, prior_psi, + sqrt(mymodel.sigma2_rot), sqrt(mymodel.sigma2_tilt), sqrt(mymodel.sigma2_psi), + exp_pointer_dir_nonzeroprior, exp_directions_prior, exp_pointer_psi_nonzeroprior, exp_psi_prior, + do_local_angular_searches, prior_psi_flip_ratio, prior_rot_flip_ratio); + } + else + { + sampling.selectOrientationsWithNonZeroPriorProbability(prior_rot, prior_tilt, prior_psi, + sqrt(mymodel.sigma2_rot), sqrt(mymodel.sigma2_tilt), sqrt(mymodel.sigma2_psi), + exp_pointer_dir_nonzeroprior, exp_directions_prior, exp_pointer_psi_nonzeroprior, exp_psi_prior, + ((do_bimodal_psi) && (mymodel.sigma2_psi > 0.)) ); + } + + long int nr_orients = sampling.NrDirections(0, &exp_pointer_dir_nonzeroprior) * sampling.NrPsiSamplings(0, &exp_pointer_psi_nonzeroprior); + if (nr_orients == 0) + { + std::cerr << " sampling.NrDirections()= " << sampling.NrDirections(0, &exp_pointer_dir_nonzeroprior) + << " sampling.NrPsiSamplings()= " << sampling.NrPsiSamplings(0, &exp_pointer_psi_nonzeroprior) << std::endl; + REPORT_ERROR("Zero orientations fall within the local angular search. Increase the sigma-value(s) on the orientations!"); + } - // If there were no defined priors (i.e. their values were 999.), then use the "normal" angles - if (prior_rot > 998.99 && prior_rot < 999.01) - prior_rot = DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_ROT); - if (prior_tilt > 998.99 && prior_tilt < 999.01) - prior_tilt = DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_TILT); - if (prior_psi > 998.99 && prior_psi < 999.01) - prior_psi = DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_PSI); - if (prior_psi_flip_ratio > 998.99 && prior_psi_flip_ratio < 999.01) - prior_psi_flip_ratio = 0.5; - if (prior_rot_flip_ratio > 998.99 && prior_rot_flip_ratio < 999.01) // Kthurber - prior_rot_flip_ratio = 0.5; // Kthurber - - // Select only those orientations that have non-zero prior probability - // Jun04,2015 - Shaoda & Sjors, bimodal psi searches for helices - if (do_helical_refine && mymodel.ref_dim == 3) - { - sampling.selectOrientationsWithNonZeroPriorProbabilityFor3DHelicalReconstruction(prior_rot, prior_tilt, prior_psi, - sqrt(mymodel.sigma2_rot), sqrt(mymodel.sigma2_tilt), sqrt(mymodel.sigma2_psi), - exp_pointer_dir_nonzeroprior, exp_directions_prior, exp_pointer_psi_nonzeroprior, exp_psi_prior, - do_local_angular_searches, prior_psi_flip_ratio, prior_rot_flip_ratio); + } + + // Helical reconstruction: calculate old_offset in the system of coordinates of the helix, i.e. parallel & perpendicular, depending on psi-angle! + // For helices do NOT apply old_offset along the direction of the helix!! + Matrix1D my_old_offset_helix_coords; + RFLOAT rot_deg = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ROT); + RFLOAT tilt_deg = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_TILT); + RFLOAT psi_deg = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PSI); + //std::cerr << " rot_deg= " << rot_deg << " tilt_deg= " << tilt_deg << " psi_deg= " << psi_deg << std::endl; + if ( (do_helical_refine) && (!ignore_helical_symmetry) ) + { + // Calculate my_old_offset_helix_coords from my_old_offset and psi angle + transformCartesianAndHelicalCoords(my_old_offset, my_old_offset_helix_coords, rot_deg, tilt_deg, psi_deg, CART_TO_HELICAL_COORDS); +#ifdef DEBUG_HELICAL_ORIENTATIONAL_SEARCH + // May 18, 2015 - Shaoda & Sjors - Helical refinement (orientational searches) + std::cerr << "MlOptimiser::getFourierTransformsAndCtfs()" << std::endl; + std::cerr << " Transform old Cartesian offsets to helical ones..." << std::endl; + if(my_old_offset.size() == 2) + { + std::cerr << " psi_deg = " << psi_deg << " degrees" << std::endl; + std::cerr << " old_offset(x, y) = (" << XX(my_old_offset) << ", " << YY(my_old_offset) << ")" << std::endl; + std::cerr << " old_offset_helix(r, p) = (" << XX(my_old_offset_helix_coords) << ", " << YY(my_old_offset_helix_coords) << ")" << std::endl; } else { - sampling.selectOrientationsWithNonZeroPriorProbability(prior_rot, prior_tilt, prior_psi, - sqrt(mymodel.sigma2_rot), sqrt(mymodel.sigma2_tilt), sqrt(mymodel.sigma2_psi), - exp_pointer_dir_nonzeroprior, exp_directions_prior, exp_pointer_psi_nonzeroprior, exp_psi_prior, - ((do_bimodal_psi) && (mymodel.sigma2_psi > 0.)) ); + std::cerr << " psi_deg = " << psi_deg << " degrees, tilt_deg = " << tilt_deg << " degrees"<< std::endl; + std::cerr << " old_offset(x, y, z) = (" << XX(my_old_offset) << ", " << YY(my_old_offset) << ", " << ZZ(my_old_offset) << ")" << std::endl; + std::cerr << " old_offset_helix(p1, p2, z) = (" << XX(my_old_offset_helix_coords) << ", " << YY(my_old_offset_helix_coords) << "," << ZZ(my_old_offset_helix_coords) << ")" << std::endl; } +#endif + // We do NOT want to accumulate the offsets in the direction along the helix (which is X in the helical coordinate system!) + // However, when doing helical local searches, we accumulate offsets + // Do NOT accumulate offsets in 3D classification of helices + if ( (!do_skip_align) && (!do_skip_rotate) ) + { + // TODO: check whether the following lines make sense + bool do_auto_refine_local_searches = (do_auto_refine || do_auto_sampling) && (sampling.healpix_order >= autosampling_hporder_local_searches); + bool do_classification_local_searches = (!do_auto_refine) && (mymodel.orientational_prior_mode == PRIOR_ROTTILT_PSI) + && (mymodel.sigma2_rot > 0.) && (mymodel.sigma2_tilt > 0.) && (mymodel.sigma2_psi > 0.); + bool do_local_angular_searches = (do_auto_refine_local_searches) || (do_classification_local_searches); + if (!do_local_angular_searches) + { + if (mymodel.data_dim == 2) + XX(my_old_offset_helix_coords) = 0.; + else if (mymodel.data_dim == 3 || mydata.is_tomo) + ZZ(my_old_offset_helix_coords) = 0.; + } + } +#ifdef DEBUG_HELICAL_ORIENTATIONAL_SEARCH + std::cerr << " Set r (translation along helical axis) to zero..." << std::endl; + if(my_old_offset.size() == 2) + std::cerr << " old_offset_helix(r, p) = (" << XX(my_old_offset_helix_coords) << ", " << YY(my_old_offset_helix_coords) << ")" << std::endl; + else + std::cerr << " old_offset_helix(p1, p2, z) = (" << XX(my_old_offset_helix_coords) << ", " << YY(my_old_offset_helix_coords) << "," << ZZ(my_old_offset_helix_coords) << ")" << std::endl; +#endif + // Now re-calculate the my_old_offset in the real (or image) system of coordinate (rotate -psi angle) + transformCartesianAndHelicalCoords(my_old_offset_helix_coords, my_old_offset, rot_deg, tilt_deg, psi_deg, HELICAL_TO_CART_COORDS); +#ifdef DEBUG_HELICAL_ORIENTATIONAL_SEARCH + std::cerr << " Transform helical offsets back to Cartesian ones..." << std::endl; + if(my_old_offset.size() == 2) + std::cerr << " old_offset(x, y) = (" << XX(my_old_offset) << ", " << YY(my_old_offset) << ")" << std::endl; + else + std::cerr << " old_offset(x, y, z) = (" << XX(my_old_offset) << ", " << YY(my_old_offset) << ", " << ZZ(my_old_offset) << ")" << std::endl; +#endif + } - long int nr_orients = sampling.NrDirections(0, &exp_pointer_dir_nonzeroprior) * sampling.NrPsiSamplings(0, &exp_pointer_psi_nonzeroprior); - if (nr_orients == 0) - { - std::cerr << " sampling.NrDirections()= " << sampling.NrDirections(0, &exp_pointer_dir_nonzeroprior) - << " sampling.NrPsiSamplings()= " << sampling.NrPsiSamplings(0, &exp_pointer_psi_nonzeroprior) << std::endl; - REPORT_ERROR("Zero orientations fall within the local angular search. Increase the sigma-value(s) on the orientations!"); - } + my_old_offset.selfROUND(); // Below, this rounded my_old_offset will be applied to the actual images + +#ifdef DEBUG_HELICAL_ORIENTATIONAL_SEARCH + if ( (do_helical_refine) && (!ignore_helical_symmetry) ) + { + std::cerr << " Apply (rounded) old offsets (r = 0, p) & (psi, tilt) for helices..." << std::endl; + if(my_old_offset.size() == 2) + std::cerr << " old_offset(x, y) = (" << XX(my_old_offset) << ", " << YY(my_old_offset) << ")" << std::endl; + else + std::cerr << " old_offset(x, y, z) = (" << XX(my_old_offset) << ", " << YY(my_old_offset) << ", " << ZZ(my_old_offset) << ")" << std::endl; + Image tt; + tt = img; + tt.write("selftranslated_helix.spi"); + tt.clear(); + std::cerr << " written selftranslated_helix.spi; press any key to continue..." << std::endl; + std::string str; + std::cin >> str; + } +#endif + if ( (do_helical_refine) && (!ignore_helical_symmetry) ) + { + // Transform rounded Cartesian offsets to corresponding helical ones + transformCartesianAndHelicalCoords(my_old_offset, my_old_offset_helix_coords, rot_deg, tilt_deg, psi_deg, CART_TO_HELICAL_COORDS); + exp_old_offset = my_old_offset_helix_coords; + } + else + { + // For multi-bodies: store only the old refined offset, not the constant consensus offset or the projected COM of this body + if (mymodel.nr_bodies > 1) + exp_old_offset = my_refined_ibody_offset; + else + exp_old_offset = my_old_offset; // Not doing helical refinement. Rounded Cartesian offsets are stored. + } +#ifdef DEBUG_HELICAL_ORIENTATIONAL_SEARCH + if ( (do_helical_refine) && (!ignore_helical_symmetry) ) + { + if(exp_old_offset[img_id].size() == 2) + std::cerr << "exp_old_offset = (" << XX(exp_old_offset[img_id]) << ", " << YY(exp_old_offset[img_id][img_id]) << ")" << std::endl; + else + std::cerr << "exp_old_offset = (" << XX(exp_old_offset[img_id]) << ", " << YY(exp_old_offset[img_id]) << ", " << ZZ(exp_old_offset[img_id]) << ")" << std::endl; } +#endif + + + FourierTransformer transformer; + for (int img_id = 0; img_id < mydata.numberOfImagesInParticle(part_id); img_id++) + { + Image img, rec_img; + MultidimArray Fimg, Faux; + MultidimArray Fctf, FstMulti; // SubtomoWeights + + // To which group do I belong? + int group_id = mydata.getGroupId(part_id, img_id); + // What is my optics group? + int optics_group = mydata.getOpticsGroup(part_id, img_id); + bool ctf_premultiplied = mydata.obsModel.getCtfPremultiplied(optics_group); + RFLOAT my_pixel_size = mydata.getOpticsPixelSize(optics_group); + int my_image_size = mydata.getOpticsImageSize(optics_group); + // Get the image and recimg data if (do_parallel_disc_io) @@ -5687,7 +5800,6 @@ void MlOptimiser::getFourierTransformsAndCtfs( { img().reshape(mydata.particles[part_id].images[img_id].img); - FOR_ALL_DIRECT_ELEMENTS_IN_MULTIDIMARRAY(mydata.particles[part_id].images[img_id].img) { DIRECT_MULTIDIM_ELEM(img(), n) = (RFLOAT)DIRECT_MULTIDIM_ELEM(mydata.particles[part_id].images[img_id].img, n); @@ -5733,7 +5845,7 @@ void MlOptimiser::getFourierTransformsAndCtfs( if (!mydata.getImageNameOnScratch(part_id, img_id, fn_img)) { std::istringstream split(exp_fn_img); - for (int i = 0; i <= my_metadata_offset; i++) + for (int i = 0; i <= imagedata_offset; i++) getline(split, fn_img); } img.read(fn_img); @@ -5741,7 +5853,7 @@ void MlOptimiser::getFourierTransformsAndCtfs( } else { - img() = exp_imgs[my_metadata_offset]; + img() = exp_imgs[imagedata_offset]; } #endif } @@ -5751,7 +5863,7 @@ void MlOptimiser::getFourierTransformsAndCtfs( FileName fn_recimg; std::istringstream split2(exp_fn_recimg); // Get the right line in the exp_fn_img string - for (int i = 0; i <= my_metadata_offset; i++) + for (int i = 0; i <= imagedata_offset; i++) getline(split2, fn_recimg); rec_img.read(fn_recimg); rec_img().setXmippOrigin(); @@ -5790,7 +5902,7 @@ void MlOptimiser::getFourierTransformsAndCtfs( img().resize(image_full_size[optics_group], image_full_size[optics_group]); FOR_ALL_DIRECT_ELEMENTS_IN_ARRAY2D(img()) { - DIRECT_A2D_ELEM(img(), i, j) = DIRECT_A3D_ELEM(exp_imagedata, my_metadata_offset, i, j); + DIRECT_A2D_ELEM(img(), i, j) = DIRECT_A3D_ELEM(exp_imagedata, imagedata_offset, i, j); } img().setXmippOrigin(); if (has_converged && do_use_reconstruct_images) @@ -5802,7 +5914,7 @@ void MlOptimiser::getFourierTransformsAndCtfs( rec_img().resize(image_full_size[optics_group], image_full_size[optics_group]); FOR_ALL_DIRECT_ELEMENTS_IN_ARRAY2D(rec_img()) { - DIRECT_A2D_ELEM(rec_img(), i, j) = DIRECT_A3D_ELEM(exp_imagedata, my_nr_particles + my_metadata_offset, i, j); + DIRECT_A2D_ELEM(rec_img(), i, j) = DIRECT_A3D_ELEM(exp_imagedata, my_nr_particles + imagedata_offset, i, j); } rec_img().setXmippOrigin(); } @@ -5827,121 +5939,16 @@ void MlOptimiser::getFourierTransformsAndCtfs( img() *= mymodel.avg_norm_correction / normcorr; } - // Helical reconstruction: calculate old_offset in the system of coordinates of the helix, i.e. parallel & perpendicular, depending on psi-angle! - // For helices do NOT apply old_offset along the direction of the helix!! - Matrix1D my_old_offset_helix_coords; - RFLOAT rot_deg = DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_ROT); - RFLOAT tilt_deg = DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_TILT); - RFLOAT psi_deg = DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_PSI); - //std::cerr << " rot_deg= " << rot_deg << " tilt_deg= " << tilt_deg << " psi_deg= " << psi_deg << std::endl; - if ( (do_helical_refine) && (!ignore_helical_symmetry) ) - { - // Calculate my_old_offset_helix_coords from my_old_offset and psi angle - transformCartesianAndHelicalCoords(my_old_offset, my_old_offset_helix_coords, rot_deg, tilt_deg, psi_deg, CART_TO_HELICAL_COORDS); -#ifdef DEBUG_HELICAL_ORIENTATIONAL_SEARCH - // May 18, 2015 - Shaoda & Sjors - Helical refinement (orientational searches) - std::cerr << "MlOptimiser::getFourierTransformsAndCtfs()" << std::endl; - std::cerr << " Transform old Cartesian offsets to helical ones..." << std::endl; - if(my_old_offset.size() == 2) - { - std::cerr << " psi_deg = " << psi_deg << " degrees" << std::endl; - std::cerr << " old_offset(x, y) = (" << XX(my_old_offset) << ", " << YY(my_old_offset) << ")" << std::endl; - std::cerr << " old_offset_helix(r, p) = (" << XX(my_old_offset_helix_coords) << ", " << YY(my_old_offset_helix_coords) << ")" << std::endl; - } - else - { - std::cerr << " psi_deg = " << psi_deg << " degrees, tilt_deg = " << tilt_deg << " degrees"<< std::endl; - std::cerr << " old_offset(x, y, z) = (" << XX(my_old_offset) << ", " << YY(my_old_offset) << ", " << ZZ(my_old_offset) << ")" << std::endl; - std::cerr << " old_offset_helix(p1, p2, z) = (" << XX(my_old_offset_helix_coords) << ", " << YY(my_old_offset_helix_coords) << "," << ZZ(my_old_offset_helix_coords) << ")" << std::endl; - } -#endif - // We do NOT want to accumulate the offsets in the direction along the helix (which is X in the helical coordinate system!) - // However, when doing helical local searches, we accumulate offsets - // Do NOT accumulate offsets in 3D classification of helices - if ( (!do_skip_align) && (!do_skip_rotate) ) - { - // TODO: check whether the following lines make sense - bool do_auto_refine_local_searches = (do_auto_refine || do_auto_sampling) && (sampling.healpix_order >= autosampling_hporder_local_searches); - bool do_classification_local_searches = (!do_auto_refine) && (mymodel.orientational_prior_mode == PRIOR_ROTTILT_PSI) - && (mymodel.sigma2_rot > 0.) && (mymodel.sigma2_tilt > 0.) && (mymodel.sigma2_psi > 0.); - bool do_local_angular_searches = (do_auto_refine_local_searches) || (do_classification_local_searches); - if (!do_local_angular_searches) - { - if (mymodel.data_dim == 2) - XX(my_old_offset_helix_coords) = 0.; - else if (mymodel.data_dim == 3) - ZZ(my_old_offset_helix_coords) = 0.; - } - } -#ifdef DEBUG_HELICAL_ORIENTATIONAL_SEARCH - std::cerr << " Set r (translation along helical axis) to zero..." << std::endl; - if(my_old_offset.size() == 2) - std::cerr << " old_offset_helix(r, p) = (" << XX(my_old_offset_helix_coords) << ", " << YY(my_old_offset_helix_coords) << ")" << std::endl; - else - std::cerr << " old_offset_helix(p1, p2, z) = (" << XX(my_old_offset_helix_coords) << ", " << YY(my_old_offset_helix_coords) << "," << ZZ(my_old_offset_helix_coords) << ")" << std::endl; -#endif - // Now re-calculate the my_old_offset in the real (or image) system of coordinate (rotate -psi angle) - transformCartesianAndHelicalCoords(my_old_offset_helix_coords, my_old_offset, rot_deg, tilt_deg, psi_deg, HELICAL_TO_CART_COORDS); -#ifdef DEBUG_HELICAL_ORIENTATIONAL_SEARCH - std::cerr << " Transform helical offsets back to Cartesian ones..." << std::endl; - if(my_old_offset.size() == 2) - std::cerr << " old_offset(x, y) = (" << XX(my_old_offset) << ", " << YY(my_old_offset) << ")" << std::endl; - else - std::cerr << " old_offset(x, y, z) = (" << XX(my_old_offset) << ", " << YY(my_old_offset) << ", " << ZZ(my_old_offset) << ")" << std::endl; -#endif - } - my_old_offset.selfROUND(); - selfTranslate(img(), my_old_offset, DONT_WRAP); + // TODO! Think this through for 2D stacks.... + if (mydata.is_tomo) REPORT_ERROR("selfTranslate will not work for 2D STA stacks!") + + selfTranslate(img(), my_old_offset, DONT_WRAP); if (has_converged && do_use_reconstruct_images) { selfTranslate(rec_img(), my_old_offset, DONT_WRAP); } -#ifdef DEBUG_HELICAL_ORIENTATIONAL_SEARCH - if ( (do_helical_refine) && (!ignore_helical_symmetry) ) - { - std::cerr << " Apply (rounded) old offsets (r = 0, p) & (psi, tilt) for helices..." << std::endl; - if(my_old_offset.size() == 2) - std::cerr << " old_offset(x, y) = (" << XX(my_old_offset) << ", " << YY(my_old_offset) << ")" << std::endl; - else - std::cerr << " old_offset(x, y, z) = (" << XX(my_old_offset) << ", " << YY(my_old_offset) << ", " << ZZ(my_old_offset) << ")" << std::endl; - Image tt; - tt = img; - tt.write("selftranslated_helix.spi"); - tt.clear(); - std::cerr << " written selftranslated_helix.spi; press any key to continue..." << std::endl; - std::string str; - std::cin >> str; - } -#endif - - if ( (do_helical_refine) && (!ignore_helical_symmetry) ) - { - // Transform rounded Cartesian offsets to corresponding helical ones - transformCartesianAndHelicalCoords(my_old_offset, my_old_offset_helix_coords, rot_deg, tilt_deg, psi_deg, CART_TO_HELICAL_COORDS); - exp_old_offset[img_id] = my_old_offset_helix_coords; - } - else - { - // For multi-bodies: store only the old refined offset, not the constant consensus offset or the projected COM of this body - if (mymodel.nr_bodies > 1) - exp_old_offset[img_id] = my_refined_ibody_offset; - else - exp_old_offset[img_id] = my_old_offset; // Not doing helical refinement. Rounded Cartesian offsets are stored. - } -#ifdef DEBUG_HELICAL_ORIENTATIONAL_SEARCH - if ( (do_helical_refine) && (!ignore_helical_symmetry) ) - { - if(exp_old_offset[img_id].size() == 2) - std::cerr << "exp_old_offset = (" << XX(exp_old_offset[img_id]) << ", " << YY(exp_old_offset[img_id][img_id]) << ")" << std::endl; - else - std::cerr << "exp_old_offset = (" << XX(exp_old_offset[img_id]) << ", " << YY(exp_old_offset[img_id]) << ", " << ZZ(exp_old_offset[img_id]) << ")" << std::endl; - } -#endif - // Also store priors on translations - exp_prior[img_id] = my_prior; - //#define DEBUG_SOFTMASK #ifdef DEBUG_SOFTMASK Image tt; @@ -6016,7 +6023,10 @@ void MlOptimiser::getFourierTransformsAndCtfs( // May24,2014 - Shaoda & Sjors, Helical refinement if (is_helical_segment) { - softMaskOutsideMapForHelix(img(), psi_deg, tilt_deg, my_mask_radius, + // TODO! Think about how the direction of the individual images in tilt series stack changes the direction of the mask for STA!! + if (mydata.is_tomo) REPORT_ERROR("ERROR: softMaskOutsideMapForHelix for STA not implemented yet..."); + + softMaskOutsideMapForHelix(img(), psi_deg, tilt_deg, my_mask_radius, (helical_tube_outer_diameter / (2. * my_pixel_size)), width_mask_edge, &Mnoise); } else @@ -6102,7 +6112,7 @@ void MlOptimiser::getFourierTransformsAndCtfs( { std::istringstream split(exp_fn_ctf); // Get the right line in the exp_fn_img string - for (int i = 0; i <= my_metadata_offset; i++) + for (int i = 0; i <= metadata_offset; i++) getline(split, fn_ctf); } Ictf.read(fn_ctf); @@ -6125,12 +6135,19 @@ void MlOptimiser::getFourierTransformsAndCtfs( CTF ctf; ctf.setValuesByGroup( &mydata.obsModel, optics_group, - DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_CTF_DEFOCUS_U), - DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_CTF_DEFOCUS_V), - DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_CTF_DEFOCUS_ANGLE), - DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_CTF_BFACTOR), - DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_CTF_KFACTOR), - DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_CTF_PHASE_SHIFT)); + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_CTF_DEFOCUS_U), + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_CTF_DEFOCUS_V), + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_CTF_DEFOCUS_ANGLE), + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_CTF_BFACTOR), + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_CTF_KFACTOR), + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_CTF_PHASE_SHIFT)); + + // Apply the dz to the defocus for 2D image stacks in STA + if (mydata.is_tomo) + { + ctf.DeltafU += mydata.particles[part_id].images[img_id].dz; + ctf.DeltafV += mydata.particles[part_id].images[img_id].dz; + } ctf.getFftwImage(Fctf, image_full_size[optics_group], image_full_size[optics_group], my_pixel_size, ctf_phase_flipped, only_flip_phases, intact_ctf_first_peak, true, do_ctf_padding); @@ -6202,9 +6219,9 @@ void MlOptimiser::getFourierTransformsAndCtfs( Matrix2D Aresi, Abody; // Aresi is the residual orientation for this obody - Euler_angles2matrix(DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, ocol_rot), - DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, ocol_tilt), - DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, ocol_psi), Aresi, false); + Euler_angles2matrix(DIRECT_A2D_ELEM(exp_metadata, metadata_offset, ocol_rot), + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, ocol_tilt), + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, ocol_psi), Aresi, false); // The real orientation to be applied is the obody transformation applied and the original one Abody = Aori * (mymodel.orient_bodies[obody]).transpose() * A_rot90 * Aresi * mymodel.orient_bodies[obody]; @@ -6269,11 +6286,11 @@ void MlOptimiser::getFourierTransformsAndCtfs( #endif // Subtract refined obody-displacement - XX(other_projected_com) -= DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, ocol_xoff); - YY(other_projected_com) -= DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, ocol_yoff); + XX(other_projected_com) -= DIRECT_A2D_ELEM(exp_metadata, metadata_offset, ocol_xoff); + YY(other_projected_com) -= DIRECT_A2D_ELEM(exp_metadata, metadata_offset, ocol_yoff); if (mymodel.data_dim == 3) { - ZZ(other_projected_com) -= DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, ocol_zoff); + ZZ(other_projected_com) -= DIRECT_A2D_ELEM(exp_metadata, metadata_offset, ocol_zoff); } // Add the my_old_offset=selfRound(my_old_offset_ori - my_projected_com) already applied to this image for ibody @@ -6285,7 +6302,11 @@ void MlOptimiser::getFourierTransformsAndCtfs( std::cerr << " obody: " << obody+1 << " APPLIED translation obody= " << other_projected_com.transpose() << std::endl; } #endif - shiftImageInFourierTransform(FTo, Faux, (RFLOAT)mymodel.ori_size, + + // TODO! The below will now work for 2D stacks in 2D, just like selfTranslate() didn't work upwards; Also need to apply per-image rotations, Aproj! + if (mydata.is_tomo) REPORT_ERROR("ERROR: multibody for STA not implemented yet!"); + + shiftImageInFourierTransform(FTo, Faux, (RFLOAT)mymodel.ori_size, XX(other_projected_com), YY(other_projected_com), (mymodel.data_dim == 3) ? ZZ(other_projected_com) : 0); // Sum the Fourier transforms of all the obodies @@ -9821,7 +9842,7 @@ void MlOptimiser::checkConvergence(bool myverb) void MlOptimiser::setMetaDataSubset(long int first_part_id, long int last_part_id) { - for (long int part_id_sorted = first_part_id, metadata_offset = 0; part_id_sorted <= last_part_id; part_id_sorted++) + for (long int part_id_sorted = first_part_id, metadata_offset = 0; part_id_sorted <= last_part_id; part_id_sorted++, metadata_offset++) { long int part_id = mydata.sorted_idx[part_id_sorted]; @@ -9959,11 +9980,11 @@ void MlOptimiser::getMetaAndImageDataSubset(long int first_part_id, long int las } } - for (long int part_id_sorted = first_part_id, metadata_offset = 0; part_id_sorted <= last_part_id; part_id_sorted++) + for (long int part_id_sorted = first_part_id, metadata_offset = 0, imagedata_offset = 0; part_id_sorted <= last_part_id; part_id_sorted++, metadata_offset++) { long int part_id = mydata.sorted_idx[part_id_sorted]; - for (int img_id = 0; img_id < mydata.numberOfImagesInParticle(part_id); img_id++, metadata_offset++) + for (int img_id = 0; img_id < mydata.numberOfImagesInParticle(part_id); img_id++, imagedata_offset++) { RFLOAT my_pixel_size = mydata.getImagePixelSize(part_id, img_id); @@ -9972,7 +9993,7 @@ void MlOptimiser::getMetaAndImageDataSubset(long int first_part_id, long int las // Get the image names from the MDimg table FileName fn_img="", fn_rec_img="", fn_ctf=""; if (!mydata.getImageNameOnScratch(part_id, img_id, fn_img)) - mydata.MDimg.getValue(EMDL_IMAGE_NAME, fn_img, part_id); + mydata.MDimg.getValue(EMDL_IMAGE_NAME, fn_img, part_id); if (mymodel.data_dim == 3 && do_ctf_correction) { @@ -10061,14 +10082,14 @@ void MlOptimiser::getMetaAndImageDataSubset(long int first_part_id, long int las { FOR_ALL_DIRECT_ELEMENTS_IN_ARRAY2D(img()) { - DIRECT_A3D_ELEM(exp_imagedata, metadata_offset, i, j) = DIRECT_A2D_ELEM(img(), i, j); + DIRECT_A3D_ELEM(exp_imagedata, imagedata_offset, i, j) = DIRECT_A2D_ELEM(img(), i, j); } if (has_converged && do_use_reconstruct_images) { FOR_ALL_DIRECT_ELEMENTS_IN_ARRAY2D(rec_img()) { - DIRECT_A3D_ELEM(exp_imagedata, metadata_offset, i, j) = DIRECT_A2D_ELEM(rec_img(), i, j); + DIRECT_A3D_ELEM(exp_imagedata, imagedata_offset, i, j) = DIRECT_A2D_ELEM(rec_img(), i, j); } } @@ -10083,135 +10104,137 @@ void MlOptimiser::getMetaAndImageDataSubset(long int first_part_id, long int las exp_fn_recimg += fn_rec_img + "\n"; } - // Now get the metadata - int iaux; - mydata.MDimg.getValue(EMDL_ORIENT_ROT, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ROT), part_id); - mydata.MDimg.getValue(EMDL_ORIENT_TILT, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_TILT), part_id); - mydata.MDimg.getValue(EMDL_ORIENT_PSI, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PSI), part_id); - RFLOAT xoff_A, yoff_A, zoff_A; - mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_X_ANGSTROM, xoff_A, part_id); - mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_Y_ANGSTROM, yoff_A, part_id); - DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_XOFF) = xoff_A / my_pixel_size; - DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_YOFF) = yoff_A / my_pixel_size; - if (mymodel.data_dim == 3) - { - mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_Z_ANGSTROM, zoff_A, part_id); - DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ZOFF) = zoff_A / my_pixel_size; - } + } // end for img_id - mydata.MDimg.getValue(EMDL_PARTICLE_CLASS, iaux, part_id); - DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_CLASS) = (RFLOAT)iaux; - mydata.MDimg.getValue(EMDL_PARTICLE_DLL, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_DLL), part_id); - mydata.MDimg.getValue(EMDL_PARTICLE_PMAX, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PMAX), part_id); + // Now get the metadata + RFLOAT my_pixel_size = mydata.getImagePixelSize(part_id, 0); // pixel size has to be the same for all images in a particle... - // 5jul17: we do not need EMDL_PARTICLE_NR_SIGNIFICANT_SAMPLES for calculations. Send randomsubset instead! - if (do_split_random_halves) - mydata.MDimg.getValue(EMDL_PARTICLE_RANDOM_SUBSET, iaux, part_id); - else - mydata.MDimg.getValue(EMDL_PARTICLE_NR_SIGNIFICANT_SAMPLES, iaux, part_id); - DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_NR_SIGN) = (RFLOAT)iaux; - if (!mydata.MDimg.getValue(EMDL_IMAGE_NORM_CORRECTION, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_NORM), part_id)) - DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_NORM) = 1.; - - // If the priors are NOT set, then set their values to 999. - if (!mydata.MDimg.getValue(EMDL_ORIENT_ROT_PRIOR, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ROT_PRIOR), part_id)) - DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ROT_PRIOR) = 999.; - if (!mydata.MDimg.getValue(EMDL_ORIENT_TILT_PRIOR, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_TILT_PRIOR), part_id)) - DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_TILT_PRIOR) = 999.; - if (!mydata.MDimg.getValue(EMDL_ORIENT_PSI_PRIOR, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PSI_PRIOR), part_id)) - DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PSI_PRIOR) = 999.; - if (mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_X_PRIOR_ANGSTROM, xoff_A, part_id)) - { - DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_XOFF_PRIOR) = xoff_A / my_pixel_size; - } - else - { - DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_XOFF_PRIOR) = 999.; - } - if (mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_Y_PRIOR_ANGSTROM, yoff_A, part_id)) - { - DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_YOFF_PRIOR) = yoff_A / my_pixel_size; - } - else - { - DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_YOFF_PRIOR) = 999.; - } - if (mymodel.data_dim == 3) - { - if (mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_Z_PRIOR_ANGSTROM, zoff_A, part_id)) - { - DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ZOFF_PRIOR) = zoff_A / my_pixel_size; - } - else - { - DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ZOFF_PRIOR) = 999.; - } - } - if (!mydata.MDimg.getValue(EMDL_ORIENT_PSI_PRIOR_FLIP_RATIO, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PSI_PRIOR_FLIP_RATIO), part_id)) - DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PSI_PRIOR_FLIP_RATIO) = 999.; + int iaux; + mydata.MDimg.getValue(EMDL_ORIENT_ROT, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ROT), part_id); + mydata.MDimg.getValue(EMDL_ORIENT_TILT, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_TILT), part_id); + mydata.MDimg.getValue(EMDL_ORIENT_PSI, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PSI), part_id); + RFLOAT xoff_A, yoff_A, zoff_A; + mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_X_ANGSTROM, xoff_A, part_id); + mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_Y_ANGSTROM, yoff_A, part_id); + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_XOFF) = xoff_A / my_pixel_size; + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_YOFF) = yoff_A / my_pixel_size; + if (mymodel.data_dim == 3) + { + mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_Z_ANGSTROM, zoff_A, part_id); + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ZOFF) = zoff_A / my_pixel_size; + } - // The following per-particle parameters are passed around through metadata - // Note beamtilt is no longer part of this: it is now in the optics group - if (do_ctf_correction) - { - RFLOAT DeltafU, DeltafV, azimuthal_angle, Bfac, kfac, phase_shift; + mydata.MDimg.getValue(EMDL_PARTICLE_CLASS, iaux, part_id); + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_CLASS) = (RFLOAT)iaux; + mydata.MDimg.getValue(EMDL_PARTICLE_DLL, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_DLL), part_id); + mydata.MDimg.getValue(EMDL_PARTICLE_PMAX, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PMAX), part_id); + + // 5jul17: we do not need EMDL_PARTICLE_NR_SIGNIFICANT_SAMPLES for calculations. Send randomsubset instead! + if (do_split_random_halves) + mydata.MDimg.getValue(EMDL_PARTICLE_RANDOM_SUBSET, iaux, part_id); + else + mydata.MDimg.getValue(EMDL_PARTICLE_NR_SIGNIFICANT_SAMPLES, iaux, part_id); + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_NR_SIGN) = (RFLOAT)iaux; + if (!mydata.MDimg.getValue(EMDL_IMAGE_NORM_CORRECTION, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_NORM), part_id)) + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_NORM) = 1.; + + // If the priors are NOT set, then set their values to 999. + if (!mydata.MDimg.getValue(EMDL_ORIENT_ROT_PRIOR, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ROT_PRIOR), part_id)) + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ROT_PRIOR) = 999.; + if (!mydata.MDimg.getValue(EMDL_ORIENT_TILT_PRIOR, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_TILT_PRIOR), part_id)) + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_TILT_PRIOR) = 999.; + if (!mydata.MDimg.getValue(EMDL_ORIENT_PSI_PRIOR, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PSI_PRIOR), part_id)) + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PSI_PRIOR) = 999.; + if (mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_X_PRIOR_ANGSTROM, xoff_A, part_id)) + { + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_XOFF_PRIOR) = xoff_A / my_pixel_size; + } + else + { + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_XOFF_PRIOR) = 999.; + } + if (mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_Y_PRIOR_ANGSTROM, yoff_A, part_id)) + { + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_YOFF_PRIOR) = yoff_A / my_pixel_size; + } + else + { + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_YOFF_PRIOR) = 999.; + } + if (mymodel.data_dim == 3) + { + if (mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_Z_PRIOR_ANGSTROM, zoff_A, part_id)) + { + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ZOFF_PRIOR) = zoff_A / my_pixel_size; + } + else + { + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ZOFF_PRIOR) = 999.; + } + } + if (!mydata.MDimg.getValue(EMDL_ORIENT_PSI_PRIOR_FLIP_RATIO, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PSI_PRIOR_FLIP_RATIO), part_id)) + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PSI_PRIOR_FLIP_RATIO) = 999.; - if (!mydata.MDimg.getValue(EMDL_CTF_DEFOCUSU, DeltafU, part_id)) - DeltafU=0; + // The following per-particle parameters are passed around through metadata + // Note beamtilt is no longer part of this: it is now in the optics group + if (do_ctf_correction) + { + RFLOAT DeltafU, DeltafV, azimuthal_angle, Bfac, kfac, phase_shift; - if (!mydata.MDimg.getValue(EMDL_CTF_DEFOCUSV, DeltafV, part_id)) - DeltafV=DeltafU; + if (!mydata.MDimg.getValue(EMDL_CTF_DEFOCUSU, DeltafU, part_id)) + DeltafU=0; - if (!mydata.MDimg.getValue(EMDL_CTF_DEFOCUS_ANGLE, azimuthal_angle, part_id)) - azimuthal_angle=0; + if (!mydata.MDimg.getValue(EMDL_CTF_DEFOCUSV, DeltafV, part_id)) + DeltafV=DeltafU; - if (!mydata.MDimg.getValue(EMDL_CTF_BFACTOR, Bfac, part_id)) - Bfac=0.; + if (!mydata.MDimg.getValue(EMDL_CTF_DEFOCUS_ANGLE, azimuthal_angle, part_id)) + azimuthal_angle=0; - if (!mydata.MDimg.getValue(EMDL_CTF_SCALEFACTOR, kfac, part_id)) - kfac=1.; + if (!mydata.MDimg.getValue(EMDL_CTF_BFACTOR, Bfac, part_id)) + Bfac=0.; - if (!mydata.MDimg.getValue(EMDL_CTF_PHASESHIFT, phase_shift, part_id)) - phase_shift=0.; + if (!mydata.MDimg.getValue(EMDL_CTF_SCALEFACTOR, kfac, part_id)) + kfac=1.; - DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_CTF_DEFOCUS_U) = DeltafU; - DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_CTF_DEFOCUS_V) = DeltafV; - DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_CTF_DEFOCUS_ANGLE) = azimuthal_angle; - DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_CTF_BFACTOR) = Bfac; - DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_CTF_KFACTOR) = kfac; - DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_CTF_PHASE_SHIFT) = phase_shift; + if (!mydata.MDimg.getValue(EMDL_CTF_PHASESHIFT, phase_shift, part_id)) + phase_shift=0.; - } + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_CTF_DEFOCUS_U) = DeltafU; + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_CTF_DEFOCUS_V) = DeltafV; + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_CTF_DEFOCUS_ANGLE) = azimuthal_angle; + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_CTF_BFACTOR) = Bfac; + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_CTF_KFACTOR) = kfac; + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_CTF_PHASE_SHIFT) = phase_shift; - // For multi-body refinement - if (mymodel.nr_bodies > 1) - { - for (int ibody = 0; ibody < mymodel.nr_bodies; ibody++) - { - int icol_rot = 0 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; - int icol_tilt = 1 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; - int icol_psi = 2 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; - int icol_xoff = 3 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; - int icol_yoff = 4 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; - int icol_zoff = 5 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; - RFLOAT rot, tilt, psi, xoff, yoff, zoff=0.; - mydata.MDbodies[ibody].getValue(EMDL_ORIENT_ROT, rot, part_id); - mydata.MDbodies[ibody].getValue(EMDL_ORIENT_TILT, tilt, part_id); - mydata.MDbodies[ibody].getValue(EMDL_ORIENT_PSI, psi, part_id); - mydata.MDbodies[ibody].getValue(EMDL_ORIENT_ORIGIN_X_ANGSTROM, xoff, part_id); - mydata.MDbodies[ibody].getValue(EMDL_ORIENT_ORIGIN_Y_ANGSTROM, yoff, part_id); - if (mymodel.data_dim == 3) - mydata.MDbodies[ibody].getValue(EMDL_ORIENT_ORIGIN_Z_ANGSTROM, zoff, part_id); - DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_rot) = rot; - DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_tilt) = tilt; - DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_psi) = psi; - DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_xoff) = xoff / my_pixel_size; - DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_yoff) = yoff / my_pixel_size; - DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_zoff) = zoff / my_pixel_size; - } - } + } - } // end for img_id + // For multi-body refinement + if (mymodel.nr_bodies > 1) + { + for (int ibody = 0; ibody < mymodel.nr_bodies; ibody++) + { + int icol_rot = 0 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; + int icol_tilt = 1 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; + int icol_psi = 2 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; + int icol_xoff = 3 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; + int icol_yoff = 4 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; + int icol_zoff = 5 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; + RFLOAT rot, tilt, psi, xoff, yoff, zoff=0.; + mydata.MDbodies[ibody].getValue(EMDL_ORIENT_ROT, rot, part_id); + mydata.MDbodies[ibody].getValue(EMDL_ORIENT_TILT, tilt, part_id); + mydata.MDbodies[ibody].getValue(EMDL_ORIENT_PSI, psi, part_id); + mydata.MDbodies[ibody].getValue(EMDL_ORIENT_ORIGIN_X_ANGSTROM, xoff, part_id); + mydata.MDbodies[ibody].getValue(EMDL_ORIENT_ORIGIN_Y_ANGSTROM, yoff, part_id); + if (mymodel.data_dim == 3) + mydata.MDbodies[ibody].getValue(EMDL_ORIENT_ORIGIN_Z_ANGSTROM, zoff, part_id); + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_rot) = rot; + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_tilt) = tilt; + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_psi) = psi; + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_xoff) = xoff / my_pixel_size; + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_yoff) = yoff / my_pixel_size; + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_zoff) = zoff / my_pixel_size; + } + } } // end for part_id diff --git a/src/ml_optimiser.h b/src/ml_optimiser.h index e77e24189..1428b90fb 100644 --- a/src/ml_optimiser.h +++ b/src/ml_optimiser.h @@ -1057,12 +1057,12 @@ class MlOptimiser /* Read image and its metadata from disc (threaded over all pooled particles) */ - void getFourierTransformsAndCtfs(long int part_id, int ibody, int metadata_offset, + void getFourierTransformsAndCtfs(long int part_id, int ibody, int metadata_offset, int imagedata_offset, std::vector > &exp_Fimg, std::vector > &exp_Fimg_nomask, std::vector > &exp_Fctf, - std::vector > &exp_old_offset, - std::vector > &exp_prior, + Matrix1D &exp_old_offset, + Matrix1D &exp_prior, std::vector > &exp_power_img, std::vector &exp_highres_Xi2_img, std::vector &exp_pointer_dir_nonzeroprior, From 2e9955ebe7dd257e70b45ff8c71aebd14a7fddca Mon Sep 17 00:00:00 2001 From: scheres Date: Wed, 6 Jul 2022 17:26:52 +0100 Subject: [PATCH 073/495] add diff2 over all img_id in getAllSquaredDifferences, legacy, GPU and CPU --- src/acc/acc_helper_functions_impl.h | 2 +- src/acc/acc_ml_optimiser.h | 7 +- src/acc/acc_ml_optimiser_impl.h | 840 +++++++++++++------------- src/acc/cpu/cpu_kernels/diff2.h | 4 +- src/acc/cuda/cuda_kernels/diff2.cuh | 2 +- src/acc/cuda/cuda_kernels/helper.cu | 2 +- src/ml_optimiser.cpp | 906 ++++++++++++++-------------- src/ml_optimiser.h | 14 +- 8 files changed, 885 insertions(+), 892 deletions(-) diff --git a/src/acc/acc_helper_functions_impl.h b/src/acc/acc_helper_functions_impl.h index 956412cf4..33468138b 100644 --- a/src/acc/acc_helper_functions_impl.h +++ b/src/acc/acc_helper_functions_impl.h @@ -829,7 +829,7 @@ void mapAllWeightsToMweights( LAUNCH_HANDLE_ERROR(cudaGetLastError()); #else for (size_t i=0; i < combinations; i++) - d_mweights[d_iorient[i/translation_num] * translation_num + i%translation_num] = + d_mweights[d_iorient[i/translation_num] * translation_num + i%translation_num] += d_allweights[i/translation_num * translation_num + i%translation_num]; // TODO - isn't this just d_allweights[idx + idx%translation_num]? Really? #endif diff --git a/src/acc/acc_ml_optimiser.h b/src/acc/acc_ml_optimiser.h index cab6c6f31..b52fcf4e2 100644 --- a/src/acc/acc_ml_optimiser.h +++ b/src/acc/acc_ml_optimiser.h @@ -140,15 +140,16 @@ class OptimisationParamters std::vector > Fctf, local_Fctf, local_Minvsigma2, FstMulti; std::vector pointer_dir_nonzeroprior, pointer_psi_nonzeroprior; std::vector directions_prior, psi_prior, local_sqrtXi2; - std::vector highres_Xi2_img, min_diff2; + std::vector highres_Xi2_img; + RFLOAT min_diff2; MultidimArray Mcoarse_significant; // And from storeWeightedSums - std::vector sum_weight, significant_weight, max_weight; + RFLOAT sum_weight, significant_weight, max_weight; std::vector< std::vector > sum_weight_class; Matrix1D old_offset, prior; std::vector > power_img; MultidimArray Mweight; - std::vector max_index; + Indices max_index; OptimisationParamters (unsigned nr_images, unsigned long part_id): metadata_offset(0), diff --git a/src/acc/acc_ml_optimiser_impl.h b/src/acc/acc_ml_optimiser_impl.h index 40f924ed3..b79aaaa54 100644 --- a/src/acc/acc_ml_optimiser_impl.h +++ b/src/acc/acc_ml_optimiser_impl.h @@ -1118,6 +1118,7 @@ void getAllSquaredDifferencesCoarse( buildCorrImage(baseMLO,op,corr_img,img_id,group_id); corr_img.cpToDevice(); + // REPORT_ERROR("TODO: Dari needs to write a function to add op.highres_Xi2_img[img_id] / 2. to all Weights, not initialise again!"); deviceInitValue(allWeights, (XFLOAT) (op.highres_Xi2_img[img_id] / 2.)); allWeights_pos = 0; @@ -1158,10 +1159,11 @@ void getAllSquaredDifferencesCoarse( do_CC, accMLO->dataIs3D); + if (accMLO->mydata.is_tomo) REPORT_ERROR("ERROR: TODO: think about img_id* multiplication below... I added += instead of = in the GPU and CPU kernels below!!!"); mapAllWeightsToMweights( ~projectorPlans[iclass].iorientclasses, &(~allWeights)[allWeights_pos], - &(~Mweight)[img_id*weightsPerPart], + &(~Mweight)[weightsPerPart], projectorPlans[iclass].orientation_num, translation_num, accMLO->classStreams[iclass] @@ -1179,10 +1181,10 @@ void getAllSquaredDifferencesCoarse( DEBUG_HANDLE_ERROR(cudaStreamSynchronize(accMLO->classStreams[exp_iclass])); DEBUG_HANDLE_ERROR(cudaStreamSynchronize(cudaStreamPerThread)); // does not appear to be NEEDED FOR NON-BLOCKING CLASS STREAMS in tests, but should be to sync against classStreams - op.min_diff2[img_id] = AccUtilities::getMinOnDevice(allWeights); - } // end loop img_id + op.min_diff2 = AccUtilities::getMinOnDevice(allWeights); + #ifdef TIMING if (op.part_id == baseMLO->exp_my_first_part_id) baseMLO->timer.toc(baseMLO->TIMING_ESP_DIFF1); @@ -1199,12 +1201,12 @@ void getAllSquaredDifferencesFine( SamplingParameters &sp, MlOptimiser *baseMLO, MlClass *accMLO, - std::vector &FinePassWeights, - std::vector > &FPCMasks, + IndexedDataArray &FinePassWeights, + std::vector< IndexedDataArrayMask > &FPCMasks, std::vector &FineProjectionData, AccPtrFactory ptrFactory, int ibody, - std::vector &bundleD2) + AccPtrBundle &bundleD2) { #ifdef TIMING if (op.part_id == baseMLO->exp_my_first_part_id) @@ -1283,7 +1285,8 @@ void getAllSquaredDifferencesFine( if ( (baseMLO->do_helical_refine) && (! baseMLO->ignore_helical_symmetry) ) { - RFLOAT rot_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_ROT); + if (baseMLO->mydata.is_tomo) REPORT_ERROR("ERROR; TODO: think about the below for 2D shifts in stacks..."); + RFLOAT rot_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_ROT); RFLOAT tilt_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_TILT); RFLOAT psi_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_PSI); transformCartesianAndHelicalCoords(xshift, yshift, zshift, xshift, yshift, zshift, rot_deg, tilt_deg, psi_deg, (accMLO->dataIs3D) ? (3) : (2), HELICAL_TO_CART_COORDS); @@ -1350,7 +1353,7 @@ void getAllSquaredDifferencesFine( for (unsigned long exp_iclass = sp.iclass_min; exp_iclass <= sp.iclass_max; exp_iclass++) { - FPCMasks[img_id][exp_iclass].weightNum=0; + FPCMasks[exp_iclass].weightNum=0; if ((baseMLO->mymodel.pdf_class[exp_iclass] > 0.) && (FineProjectionData[img_id].class_entries[exp_iclass] > 0) ) { @@ -1371,8 +1374,8 @@ void getAllSquaredDifferencesFine( long int nr_over_orient = baseMLO->sampling.oversamplingFactorOrientations(sp.current_oversampling); long int nr_over_trans = baseMLO->sampling.oversamplingFactorTranslations(sp.current_oversampling); // Prepare the mask of the weight-array for this class - if (FPCMasks[img_id][exp_iclass].weightNum==0) - FPCMasks[img_id][exp_iclass].firstPos = newDataSize; + if (FPCMasks[exp_iclass].weightNum==0) + FPCMasks[exp_iclass].firstPos = newDataSize; long unsigned ihidden(0); std::vector< long unsigned > iover_transes, ihiddens; @@ -1400,19 +1403,19 @@ void getAllSquaredDifferencesFine( thisClassProjectionData, iover_transes, ihiddens, nr_over_orient, nr_over_trans, img_id, - FinePassWeights[img_id], - FPCMasks[img_id][exp_iclass], // ..and output into index-arrays mask... + FinePassWeights, + FPCMasks[exp_iclass], // ..and output into index-arrays mask... chunkSize); // ..based on a given maximum chunk-size // extend size by number of significants found this class newDataSize += significant_num; - FPCMasks[img_id][exp_iclass].weightNum = significant_num; - FPCMasks[img_id][exp_iclass].lastPos = FPCMasks[img_id][exp_iclass].firstPos + significant_num; + FPCMasks[exp_iclass].weightNum = significant_num; + FPCMasks[exp_iclass].lastPos = FPCMasks[exp_iclass].firstPos + significant_num; CTOC(accMLO->timer,"pair_list_1"); CTIC(accMLO->timer,"IndexedArrayMemCp2"); - bundleD2[img_id].pack(FPCMasks[img_id][exp_iclass].jobOrigin); - bundleD2[img_id].pack(FPCMasks[img_id][exp_iclass].jobExtent); + bundleD2.pack(FPCMasks[exp_iclass].jobOrigin); + bundleD2.pack(FPCMasks[exp_iclass].jobExtent); CTOC(accMLO->timer,"IndexedArrayMemCp2"); Matrix2D MBL, MBR; @@ -1456,12 +1459,12 @@ void getAllSquaredDifferencesFine( } } - bundleD2[img_id].cpToDevice(); + bundleD2.cpToDevice(); AllEulers.cpToDevice(); - FinePassWeights[img_id].rot_id.cpToDevice(); //FIXME this is not used - FinePassWeights[img_id].rot_idx.cpToDevice(); - FinePassWeights[img_id].trans_idx.cpToDevice(); + FinePassWeights.rot_id.cpToDevice(); //FIXME this is not used + FinePassWeights.rot_idx.cpToDevice(); + FinePassWeights.trans_idx.cpToDevice(); for (unsigned long exp_iclass = sp.iclass_min; exp_iclass <= sp.iclass_max; exp_iclass++) DEBUG_HANDLE_ERROR(cudaStreamSynchronize(accMLO->classStreams[exp_iclass])); @@ -1479,7 +1482,7 @@ void getAllSquaredDifferencesFine( if(orientation_num==0) continue; - long unsigned significant_num(FPCMasks[img_id][iclass].weightNum); + long unsigned significant_num(FPCMasks[iclass].weightNum); if(significant_num==0) continue; @@ -1493,7 +1496,7 @@ void getAllSquaredDifferencesFine( CTOC(accMLO->timer,"Diff2MakeKernel"); // Use the constructed mask to construct a partial class-specific input - IndexedDataArray thisClassFinePassWeights(FinePassWeights[img_id],FPCMasks[img_id][iclass]); + IndexedDataArray thisClassFinePassWeights(FinePassWeights,FPCMasks[iclass]); CTIC(accMLO->timer,"Diff2CALL"); @@ -1509,8 +1512,8 @@ void getAllSquaredDifferencesFine( ~thisClassFinePassWeights.rot_id, ~thisClassFinePassWeights.rot_idx, ~thisClassFinePassWeights.trans_idx, - ~FPCMasks[img_id][iclass].jobOrigin, - ~FPCMasks[img_id][iclass].jobExtent, + ~FPCMasks[iclass].jobOrigin, + ~FPCMasks[iclass].jobExtent, ~thisClassFinePassWeights.weights, op, baseMLO, @@ -1521,7 +1524,7 @@ void getAllSquaredDifferencesFine( img_id, iclass, accMLO->classStreams[iclass], - FPCMasks[img_id][iclass].jobOrigin.getSize(), + FPCMasks[iclass].jobOrigin.getSize(), ((baseMLO->iter == 1 && baseMLO->do_firstiter_cc) || baseMLO->do_always_cc), accMLO->dataIs3D ); @@ -1536,17 +1539,19 @@ void getAllSquaredDifferencesFine( DEBUG_HANDLE_ERROR(cudaStreamSynchronize(accMLO->classStreams[exp_iclass])); DEBUG_HANDLE_ERROR(cudaStreamSynchronize(cudaStreamPerThread)); - FinePassWeights[img_id].setDataSize( newDataSize ); + FinePassWeights.setDataSize( newDataSize ); - CTIC(accMLO->timer,"collect_data_1"); - if(baseMLO->adaptive_oversampling!=0) - { - op.min_diff2[img_id] = (RFLOAT) AccUtilities::getMinOnDevice(FinePassWeights[img_id].weights); - } - CTOC(accMLO->timer,"collect_data_1"); + }// end loop img_id + + CTIC(accMLO->timer,"collect_data_1"); + // SHWS6July2022: only search for smallest diff2 after all img_id have been summed + if(baseMLO->adaptive_oversampling!=0) + { + op.min_diff2 = (RFLOAT) AccUtilities::getMinOnDevice(FinePassWeights.weights); + } + CTOC(accMLO->timer,"collect_data_1"); // std::cerr << " fine pass minweight = " << op.min_diff2[img_id] << std::endl; - }// end loop img_id #ifdef TIMING if (op.part_id == baseMLO->exp_my_first_part_id) baseMLO->timer.toc(baseMLO->TIMING_ESP_DIFF2); @@ -1562,8 +1567,8 @@ void convertAllSquaredDifferencesToWeights(unsigned exp_ipass, SamplingParameters &sp, MlOptimiser *baseMLO, MlClass *accMLO, - std::vector< IndexedDataArray > &PassWeights, - std::vector< std::vector< IndexedDataArrayMask > > &FPCMasks, + IndexedDataArray &PassWeights, + std::vector< IndexedDataArrayMask > &FPCMasks, AccPtr &Mweight, // FPCMasks = Fine-Pass Class-Masks AccPtrFactory ptrFactory, int ibody) @@ -1619,481 +1624,472 @@ void convertAllSquaredDifferencesToWeights(unsigned exp_ipass, if(exp_ipass==0 || baseMLO->adaptive_oversampling!=0) { - op.sum_weight.clear(); - op.sum_weight.resize(sp.nr_images, (RFLOAT)(sp.nr_images)); - op.max_weight.clear(); - op.max_weight.resize(sp.nr_images, (RFLOAT)-1); + op.sum_weight = 0.; + op.max_weight = (RFLOAT)-1.; } if (exp_ipass==0) - op.Mcoarse_significant.resizeNoCp(1,1,sp.nr_images, XSIZE(op.Mweight)); + op.Mcoarse_significant.resizeNoCp(1,1,1, XSIZE(op.Mweight)); XFLOAT my_significant_weight; - op.significant_weight.clear(); - op.significant_weight.resize(sp.nr_images, 0.); + op.significant_weight = 0.; - // loop over all images inside this particle - for (int img_id = 0; img_id < sp.nr_images; img_id++) - { - RFLOAT my_pixel_size = baseMLO->mydata.getImagePixelSize(op.part_id, img_id); + RFLOAT my_pixel_size = baseMLO->mydata.getImagePixelSize(op.part_id, 0); - RFLOAT old_offset_x, old_offset_y, old_offset_z; + RFLOAT old_offset_x, old_offset_y, old_offset_z; - if (baseMLO->mymodel.nr_bodies > 1) - { - old_offset_x = old_offset_y = old_offset_z = 0.; - } - else - { - old_offset_x = XX(op.old_offset[img_id]); - old_offset_y = YY(op.old_offset[img_id]); - if (accMLO->dataIs3D) - old_offset_z = ZZ(op.old_offset[img_id]); - } + if (baseMLO->mymodel.nr_bodies > 1) + { + old_offset_x = old_offset_y = old_offset_z = 0.; + } + else + { + old_offset_x = XX(op.old_offset); + old_offset_y = YY(op.old_offset); + if (accMLO->dataIs3D) + old_offset_z = ZZ(op.old_offset); + } - if ((baseMLO->iter == 1 && baseMLO->do_firstiter_cc) || baseMLO->do_always_cc) - { - if(exp_ipass==0) - { - int nr_coarse_weights = (sp.iclass_max-sp.iclass_min+1)*sp.nr_images * sp.nr_dir * sp.nr_psi * sp.nr_trans; - PassWeights[img_id].weights.setAccPtr(&(~Mweight)[img_id*nr_coarse_weights]); - PassWeights[img_id].weights.setHostPtr(&Mweight[img_id*nr_coarse_weights]); - PassWeights[img_id].weights.setSize(nr_coarse_weights); - } - PassWeights[img_id].weights.doFreeHost=false; + if ((baseMLO->iter == 1 && baseMLO->do_firstiter_cc) || baseMLO->do_always_cc) + { + if(exp_ipass==0) + { + int nr_coarse_weights = (sp.iclass_max-sp.iclass_min+1)*sp.nr_images * sp.nr_dir * sp.nr_psi * sp.nr_trans; + PassWeights.weights.setAccPtr(&(~Mweight)[nr_coarse_weights]); + PassWeights.weights.setHostPtr(&Mweight[nr_coarse_weights]); + PassWeights.weights.setSize(nr_coarse_weights); + } + PassWeights.weights.doFreeHost=false; - std::pair min_pair=AccUtilities::getArgMinOnDevice(PassWeights[img_id].weights); - PassWeights[img_id].weights.cpToHost(); - DEBUG_HANDLE_ERROR(cudaStreamSynchronize(cudaStreamPerThread)); + std::pair min_pair=AccUtilities::getArgMinOnDevice(PassWeights.weights); + PassWeights.weights.cpToHost(); + DEBUG_HANDLE_ERROR(cudaStreamSynchronize(cudaStreamPerThread)); - //Set all device-located weights to zero, and only the smallest one to 1. + //Set all device-located weights to zero, and only the smallest one to 1. #ifdef _CUDA_ENABLED - DEBUG_HANDLE_ERROR(cudaMemsetAsync(~(PassWeights[img_id].weights), 0.f, PassWeights[img_id].weights.getSize()*sizeof(XFLOAT),0)); + DEBUG_HANDLE_ERROR(cudaMemsetAsync(~(PassWeights.weights), 0.f, PassWeights.weights.getSize()*sizeof(XFLOAT),0)); - XFLOAT unity=1; - DEBUG_HANDLE_ERROR(cudaMemcpyAsync( &(PassWeights[img_id].weights(min_pair.first) ), &unity, sizeof(XFLOAT), cudaMemcpyHostToDevice, 0)); + XFLOAT unity=1; + DEBUG_HANDLE_ERROR(cudaMemcpyAsync( &(PassWeights.weights(min_pair.first) ), &unity, sizeof(XFLOAT), cudaMemcpyHostToDevice, 0)); - PassWeights[img_id].weights.cpToHost(); - DEBUG_HANDLE_ERROR(cudaStreamSynchronize(cudaStreamPerThread)); + PassWeights.weights.cpToHost(); + DEBUG_HANDLE_ERROR(cudaStreamSynchronize(cudaStreamPerThread)); #else - deviceInitValue(PassWeights[img_id].weights, (XFLOAT)0.0); - PassWeights[img_id].weights[min_pair.first] = (XFLOAT)1.0; + deviceInitValue(PassWeights[img_id].weights, (XFLOAT)0.0); + PassWeights[img_id].weights[min_pair.first] = (XFLOAT)1.0; #endif - my_significant_weight = 0.999; - DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_NR_SIGN) = (RFLOAT) 1.; - if (exp_ipass==0) // TODO better memset, 0 => false , 1 => true - for (int ihidden = 0; ihidden < XSIZE(op.Mcoarse_significant); ihidden++) - if (DIRECT_A2D_ELEM(op.Mweight, img_id, ihidden) >= my_significant_weight) - DIRECT_A2D_ELEM(op.Mcoarse_significant, img_id, ihidden) = true; - else - DIRECT_A2D_ELEM(op.Mcoarse_significant, img_id, ihidden) = false; - else - { - std::pair max_pair = AccUtilities::getArgMaxOnDevice(PassWeights[img_id].weights); - op.max_index[img_id].fineIdx = PassWeights[img_id].ihidden_overs[max_pair.first]; - op.max_weight[img_id] = max_pair.second; - } + my_significant_weight = 0.999; + DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_NR_SIGN) = (RFLOAT) 1.; + if (exp_ipass==0) // TODO better memset, 0 => false , 1 => true + for (int ihidden = 0; ihidden < XSIZE(op.Mcoarse_significant); ihidden++) + if (DIRECT_A1D_ELEM(op.Mweight, ihidden) >= my_significant_weight) + DIRECT_A1D_ELEM(op.Mcoarse_significant, ihidden) = true; + else + DIRECT_A1D_ELEM(op.Mcoarse_significant, ihidden) = false; + else + { + std::pair max_pair = AccUtilities::getArgMaxOnDevice(PassWeights.weights); + op.max_index.fineIdx = PassWeights.ihidden_overs[max_pair.first]; + op.max_weight = max_pair.second; + } - } - else - { + } + else + { - long int sumRedSize=0; - for (unsigned long exp_iclass = sp.iclass_min; exp_iclass <= sp.iclass_max; exp_iclass++) - sumRedSize+= (exp_ipass==0) ? ceilf((float)(sp.nr_dir*sp.nr_psi)/(float)SUMW_BLOCK_SIZE) : ceil((float)FPCMasks[img_id][exp_iclass].jobNum / (float)SUMW_BLOCK_SIZE); + long int sumRedSize=0; + for (unsigned long exp_iclass = sp.iclass_min; exp_iclass <= sp.iclass_max; exp_iclass++) + sumRedSize+= (exp_ipass==0) ? ceilf((float)(sp.nr_dir*sp.nr_psi)/(float)SUMW_BLOCK_SIZE) : ceil((float)FPCMasks[exp_iclass].jobNum / (float)SUMW_BLOCK_SIZE); - // loop through making translational priors for all classes this img_id - then copy all at once - then loop through kernel calls ( TODO: group kernel calls into one big kernel) - CTIC(accMLO->timer,"get_offset_priors"); + // loop through making translational priors for all classes this img_id - then copy all at once - then loop through kernel calls ( TODO: group kernel calls into one big kernel) + CTIC(accMLO->timer,"get_offset_priors"); - for (unsigned long exp_iclass = sp.iclass_min; exp_iclass <= sp.iclass_max; exp_iclass++) - { - RFLOAT myprior_x, myprior_y, myprior_z; - if (baseMLO->mymodel.nr_bodies > 1) - { - myprior_x = myprior_y = myprior_z = 0.; - } - else if (baseMLO->mymodel.ref_dim == 2 && !baseMLO->do_helical_refine) - { - myprior_x = XX(baseMLO->mymodel.prior_offset_class[exp_iclass]); - myprior_y = YY(baseMLO->mymodel.prior_offset_class[exp_iclass]); - } - else - { - myprior_x = XX(op.prior[img_id]); - myprior_y = YY(op.prior[img_id]); - if (accMLO->dataIs3D) - myprior_z = ZZ(op.prior[img_id]); - } + for (unsigned long exp_iclass = sp.iclass_min; exp_iclass <= sp.iclass_max; exp_iclass++) + { + RFLOAT myprior_x, myprior_y, myprior_z; + if (baseMLO->mymodel.nr_bodies > 1) + { + myprior_x = myprior_y = myprior_z = 0.; + } + else if (baseMLO->mymodel.ref_dim == 2 && !baseMLO->do_helical_refine) + { + myprior_x = XX(baseMLO->mymodel.prior_offset_class[exp_iclass]); + myprior_y = YY(baseMLO->mymodel.prior_offset_class[exp_iclass]); + } + else + { + myprior_x = XX(op.prior); + myprior_y = YY(op.prior); + if (accMLO->dataIs3D) + myprior_z = ZZ(op.prior); + } - for (unsigned long itrans = sp.itrans_min; itrans <= sp.itrans_max; itrans++) - { + for (unsigned long itrans = sp.itrans_min; itrans <= sp.itrans_max; itrans++) + { - // If it is doing helical refinement AND Cartesian vector myprior has a length > 0, transform the vector to its helical coordinates - if ( (baseMLO->do_helical_refine) && (! baseMLO->ignore_helical_symmetry)) - { - RFLOAT mypriors_len2 = myprior_x * myprior_x + myprior_y * myprior_y; - if (accMLO->dataIs3D) - mypriors_len2 += myprior_z * myprior_z; - - if (mypriors_len2 > 0.00001) - { - RFLOAT rot_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_ROT); - RFLOAT tilt_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_TILT); - RFLOAT psi_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_PSI); - transformCartesianAndHelicalCoords(myprior_x, myprior_y, myprior_z, myprior_x, myprior_y, myprior_z, rot_deg, tilt_deg, psi_deg, (accMLO->dataIs3D) ? (3) : (2), CART_TO_HELICAL_COORDS); - } - } - // (For helical refinement) Now offset, old_offset, sampling.translations and myprior are all in helical coordinates - - // To speed things up, only calculate pdf_offset at the coarse sampling. - // That should not matter much, and that way one does not need to calculate all the OversampledTranslations - double pdf(0), pdf_zeros(0); - RFLOAT offset_x = old_offset_x + baseMLO->sampling.translations_x[itrans]; - RFLOAT offset_y = old_offset_y + baseMLO->sampling.translations_y[itrans]; - double tdiff2 = 0.; - - if ( (! baseMLO->do_helical_refine) || (baseMLO->ignore_helical_symmetry) || (accMLO->dataIs3D) ) - tdiff2 += (offset_x - myprior_x) * (offset_x - myprior_x); - tdiff2 += (offset_y - myprior_y) * (offset_y - myprior_y); - if (accMLO->dataIs3D) - { - RFLOAT offset_z = old_offset_z + baseMLO->sampling.translations_z[itrans]; - if ( (! baseMLO->do_helical_refine) || (baseMLO->ignore_helical_symmetry) ) - tdiff2 += (offset_z - myprior_z) * (offset_z - myprior_z); - } + // If it is doing helical refinement AND Cartesian vector myprior has a length > 0, transform the vector to its helical coordinates + if ( (baseMLO->do_helical_refine) && (! baseMLO->ignore_helical_symmetry)) + { + RFLOAT mypriors_len2 = myprior_x * myprior_x + myprior_y * myprior_y; + if (accMLO->dataIs3D) + mypriors_len2 += myprior_z * myprior_z; - // As of version 3.1, sigma_offsets are in Angstroms! - tdiff2 *= my_pixel_size * my_pixel_size; + if (mypriors_len2 > 0.00001) + { + RFLOAT rot_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_ROT); + RFLOAT tilt_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_TILT); + RFLOAT psi_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_PSI); + transformCartesianAndHelicalCoords(myprior_x, myprior_y, myprior_z, myprior_x, myprior_y, myprior_z, rot_deg, tilt_deg, psi_deg, (accMLO->dataIs3D) ? (3) : (2), CART_TO_HELICAL_COORDS); + } + } + // (For helical refinement) Now offset, old_offset, sampling.translations and myprior are all in helical coordinates + + // To speed things up, only calculate pdf_offset at the coarse sampling. + // That should not matter much, and that way one does not need to calculate all the OversampledTranslations + double pdf(0), pdf_zeros(0); + RFLOAT offset_x = old_offset_x + baseMLO->sampling.translations_x[itrans]; + RFLOAT offset_y = old_offset_y + baseMLO->sampling.translations_y[itrans]; + double tdiff2 = 0.; + + if ( (! baseMLO->do_helical_refine) || (baseMLO->ignore_helical_symmetry) || (accMLO->dataIs3D) ) + tdiff2 += (offset_x - myprior_x) * (offset_x - myprior_x); + tdiff2 += (offset_y - myprior_y) * (offset_y - myprior_y); + if (accMLO->dataIs3D) + { + RFLOAT offset_z = old_offset_z + baseMLO->sampling.translations_z[itrans]; + if ( (! baseMLO->do_helical_refine) || (baseMLO->ignore_helical_symmetry) ) + tdiff2 += (offset_z - myprior_z) * (offset_z - myprior_z); + } - // P(offset|sigma2_offset) - // This is the probability of the offset, given the model offset and variance. - if (my_sigma2_offset < 0.0001) - { - pdf_zeros = tdiff2 > 0.; - pdf = pdf_zeros ? 0. : 1.; + // As of version 3.1, sigma_offsets are in Angstroms! + tdiff2 *= my_pixel_size * my_pixel_size; - } - else - { - pdf_zeros = false; - pdf = tdiff2 / (-2. * my_sigma2_offset); - } + // P(offset|sigma2_offset) + // This is the probability of the offset, given the model offset and variance. + if (my_sigma2_offset < 0.0001) + { + pdf_zeros = tdiff2 > 0.; + pdf = pdf_zeros ? 0. : 1.; - pdf_offset_zeros[(exp_iclass-sp.iclass_min)*sp.nr_trans + itrans] = pdf_zeros; - pdf_offset [(exp_iclass-sp.iclass_min)*sp.nr_trans + itrans] = pdf; - } - } + } + else + { + pdf_zeros = false; + pdf = tdiff2 / (-2. * my_sigma2_offset); + } - pdf_offset_zeros.cpToDevice(); - pdf_offset.cpToDevice(); + pdf_offset_zeros[(exp_iclass-sp.iclass_min)*sp.nr_trans + itrans] = pdf_zeros; + pdf_offset [(exp_iclass-sp.iclass_min)*sp.nr_trans + itrans] = pdf; + } + } - CTOC(accMLO->timer,"get_offset_priors"); - CTIC(accMLO->timer,"sumweight1"); + pdf_offset_zeros.cpToDevice(); + pdf_offset.cpToDevice(); - if(exp_ipass==0) - { - AccPtr ipartMweight( - Mweight, - img_id * op.Mweight.xdim + sp.nr_dir * sp.nr_psi * sp.nr_trans * sp.iclass_min, - (sp.iclass_max-sp.iclass_min+1) * sp.nr_dir * sp.nr_psi * sp.nr_trans); + CTOC(accMLO->timer,"get_offset_priors"); + CTIC(accMLO->timer,"sumweight1"); - pdf_offset.streamSync(); + if(exp_ipass==0) + { + AccPtr ipartMweight( + Mweight, sp.nr_dir * sp.nr_psi * sp.nr_trans * sp.iclass_min, + (sp.iclass_max-sp.iclass_min+1) * sp.nr_dir * sp.nr_psi * sp.nr_trans); - AccUtilities::kernel_weights_exponent_coarse( - sp.iclass_max-sp.iclass_min+1, - pdf_orientation, - pdf_orientation_zeros, - pdf_offset, - pdf_offset_zeros, - ipartMweight, - (XFLOAT)op.min_diff2[img_id], - sp.nr_dir*sp.nr_psi, - sp.nr_trans); + pdf_offset.streamSync(); + AccUtilities::kernel_weights_exponent_coarse( + sp.iclass_max-sp.iclass_min+1, + pdf_orientation, + pdf_orientation_zeros, + pdf_offset, + pdf_offset_zeros, + ipartMweight, + (XFLOAT)op.min_diff2, + sp.nr_dir*sp.nr_psi, + sp.nr_trans); - XFLOAT weights_max = AccUtilities::getMaxOnDevice(ipartMweight); - /* - * Add 50 since we want to stay away from e^88, which approaches the single precision limit. - * We still want as high numbers as possible to utilize most of the single precision span. - * Dari - 201710 - */ - AccUtilities::kernel_exponentiate( ipartMweight, 50 - weights_max); + XFLOAT weights_max = AccUtilities::getMaxOnDevice(ipartMweight); - CTIC(accMLO->timer,"sort"); - DEBUG_HANDLE_ERROR(cudaStreamSynchronize(cudaStreamPerThread)); + /* + * Add 50 since we want to stay away from e^88, which approaches the single precision limit. + * We still want as high numbers as possible to utilize most of the single precision span. + * Dari - 201710 + */ + AccUtilities::kernel_exponentiate( ipartMweight, 50 - weights_max); - unsigned long ipart_length = (sp.iclass_max-sp.iclass_min+1) * sp.nr_dir * sp.nr_psi * sp.nr_trans; - size_t offset = img_id * op.Mweight.xdim + sp.nr_dir * sp.nr_psi * sp.nr_trans * sp.iclass_min; + CTIC(accMLO->timer,"sort"); + DEBUG_HANDLE_ERROR(cudaStreamSynchronize(cudaStreamPerThread)); - if (ipart_length > 1) - { - //Wrap the current ipart data in a new pointer - AccPtr unsorted_ipart( - Mweight, - offset, - ipart_length); + unsigned long ipart_length = (sp.iclass_max-sp.iclass_min+1) * sp.nr_dir * sp.nr_psi * sp.nr_trans; + size_t offset = sp.nr_dir * sp.nr_psi * sp.nr_trans * sp.iclass_min; + + if (ipart_length > 1) + { + //Wrap the current ipart data in a new pointer + AccPtr unsorted_ipart( + Mweight, + offset, + ipart_length); - AccPtr filtered = ptrFactory.make((size_t)unsorted_ipart.getSize()); + AccPtr filtered = ptrFactory.make((size_t)unsorted_ipart.getSize()); - CUSTOM_ALLOCATOR_REGION_NAME("CASDTW_SORTSUM"); + CUSTOM_ALLOCATOR_REGION_NAME("CASDTW_SORTSUM"); - filtered.deviceAlloc(); + filtered.deviceAlloc(); #ifdef DEBUG_CUDA - if (unsorted_ipart.getSize()==0) - ACC_PTR_DEBUG_FATAL("Unsorted array size zero.\n"); // Hopefully Impossible + if (unsorted_ipart.getSize()==0) + ACC_PTR_DEBUG_FATAL("Unsorted array size zero.\n"); // Hopefully Impossible #endif - size_t filteredSize = AccUtilities::filterGreaterZeroOnDevice(unsorted_ipart, filtered); + size_t filteredSize = AccUtilities::filterGreaterZeroOnDevice(unsorted_ipart, filtered); - if (filteredSize == 0) - { - std::cerr << std::endl; - std::cerr << " fn_img= " << sp.current_img << std::endl; - std::cerr << " img_id= " << img_id << " adaptive_fraction= " << baseMLO->adaptive_fraction << std::endl; - std::cerr << " min_diff2= " << op.min_diff2[img_id] << std::endl; + if (filteredSize == 0) + { + std::cerr << std::endl; + std::cerr << " fn_img= " << sp.current_img << std::endl; + std::cerr " adaptive_fraction= " << baseMLO->adaptive_fraction << std::endl; + std::cerr << " min_diff2= " << op.min_diff2 << std::endl; - pdf_orientation.dumpAccToFile("error_dump_pdf_orientation"); - pdf_offset.dumpAccToFile("error_dump_pdf_offset"); - unsorted_ipart.dumpAccToFile("error_dump_filtered"); + pdf_orientation.dumpAccToFile("error_dump_pdf_orientation"); + pdf_offset.dumpAccToFile("error_dump_pdf_offset"); + unsorted_ipart.dumpAccToFile("error_dump_filtered"); - std::cerr << "Dumped data: error_dump_pdf_orientation, error_dump_pdf_orientation and error_dump_unsorted." << std::endl; + std::cerr << "Dumped data: error_dump_pdf_orientation, error_dump_pdf_orientation and error_dump_unsorted." << std::endl; - CRITICAL(ERRFILTEREDZERO); // "filteredSize == 0" - } - filtered.setSize(filteredSize); + CRITICAL(ERRFILTEREDZERO); // "filteredSize == 0" + } + filtered.setSize(filteredSize); - AccPtr sorted = ptrFactory.make((size_t)filteredSize); - AccPtr cumulative_sum = ptrFactory.make((size_t)filteredSize); + AccPtr sorted = ptrFactory.make((size_t)filteredSize); + AccPtr cumulative_sum = ptrFactory.make((size_t)filteredSize); - sorted.accAlloc(); - cumulative_sum.accAlloc(); + sorted.accAlloc(); + cumulative_sum.accAlloc(); - AccUtilities::sortOnDevice(filtered, sorted); - AccUtilities::scanOnDevice(sorted, cumulative_sum); + AccUtilities::sortOnDevice(filtered, sorted); + AccUtilities::scanOnDevice(sorted, cumulative_sum); - CTOC(accMLO->timer,"sort"); + CTOC(accMLO->timer,"sort"); - op.sum_weight[img_id] = cumulative_sum.getAccValueAt(cumulative_sum.getSize() - 1); + op.sum_weight = cumulative_sum.getAccValueAt(cumulative_sum.getSize() - 1); - long int my_nr_significant_coarse_samples; - size_t thresholdIdx = findThresholdIdxInCumulativeSum(cumulative_sum, - (1 - baseMLO->adaptive_fraction) * op.sum_weight[img_id]); + long int my_nr_significant_coarse_samples; + size_t thresholdIdx = findThresholdIdxInCumulativeSum(cumulative_sum, + (1 - baseMLO->adaptive_fraction) * op.sum_weight); - my_nr_significant_coarse_samples = filteredSize - thresholdIdx; + my_nr_significant_coarse_samples = filteredSize - thresholdIdx; - if (my_nr_significant_coarse_samples == 0) - { - std::cerr << std::endl; - std::cerr << " fn_img= " << sp.current_img << std::endl; - std::cerr << " img_id= " << img_id << " adaptive_fraction= " << baseMLO->adaptive_fraction << std::endl; - std::cerr << " threshold= " << (1 - baseMLO->adaptive_fraction) * op.sum_weight[img_id] << " thresholdIdx= " << thresholdIdx << std::endl; - std::cerr << " op.sum_weight[img_id]= " << op.sum_weight[img_id] << std::endl; - std::cerr << " min_diff2= " << op.min_diff2[img_id] << std::endl; + if (my_nr_significant_coarse_samples == 0) + { + std::cerr << std::endl; + std::cerr << " fn_img= " << sp.current_img << std::endl; + std::cerr << " adaptive_fraction= " << baseMLO->adaptive_fraction << std::endl; + std::cerr << " threshold= " << (1 - baseMLO->adaptive_fraction) * op.sum_weight << " thresholdIdx= " << thresholdIdx << std::endl; + std::cerr << " op.sum_weight[img_id]= " << op.sum_weight << std::endl; + std::cerr << " min_diff2= " << op.min_diff2 << std::endl; - unsorted_ipart.dumpAccToFile("error_dump_unsorted"); - filtered.dumpAccToFile("error_dump_filtered"); - sorted.dumpAccToFile("error_dump_sorted"); - cumulative_sum.dumpAccToFile("error_dump_cumulative_sum"); + unsorted_ipart.dumpAccToFile("error_dump_unsorted"); + filtered.dumpAccToFile("error_dump_filtered"); + sorted.dumpAccToFile("error_dump_sorted"); + cumulative_sum.dumpAccToFile("error_dump_cumulative_sum"); - std::cerr << "Written error_dump_unsorted, error_dump_filtered, error_dump_sorted, and error_dump_cumulative_sum." << std::endl; + std::cerr << "Written error_dump_unsorted, error_dump_filtered, error_dump_sorted, and error_dump_cumulative_sum." << std::endl; - CRITICAL(ERRNOSIGNIFS); // "my_nr_significant_coarse_samples == 0" - } + CRITICAL(ERRNOSIGNIFS); // "my_nr_significant_coarse_samples == 0" + } - if (baseMLO->maximum_significants > 0 && - my_nr_significant_coarse_samples > baseMLO->maximum_significants) - { - my_nr_significant_coarse_samples = baseMLO->maximum_significants; - thresholdIdx = filteredSize - my_nr_significant_coarse_samples; - } + if (baseMLO->maximum_significants > 0 && + my_nr_significant_coarse_samples > baseMLO->maximum_significants) + { + my_nr_significant_coarse_samples = baseMLO->maximum_significants; + thresholdIdx = filteredSize - my_nr_significant_coarse_samples; + } - XFLOAT significant_weight = sorted.getAccValueAt(thresholdIdx); + XFLOAT significant_weight = sorted.getAccValueAt(thresholdIdx); - CTIC(accMLO->timer,"getArgMaxOnDevice"); - std::pair max_pair = AccUtilities::getArgMaxOnDevice(unsorted_ipart); - CTOC(accMLO->timer,"getArgMaxOnDevice"); - op.max_index[img_id].coarseIdx = max_pair.first; - op.max_weight[img_id] = max_pair.second; + CTIC(accMLO->timer,"getArgMaxOnDevice"); + std::pair max_pair = AccUtilities::getArgMaxOnDevice(unsorted_ipart); + CTOC(accMLO->timer,"getArgMaxOnDevice"); + op.max_index.coarseIdx = max_pair.first; + op.max_weight = max_pair.second; - // Store nr_significant_coarse_samples for this particle - // Don't do this for multibody, as it would be overwritten for each body, - // and we also use METADATA_NR_SIGN in the new safeguard for the gold-standard separation - if (baseMLO->mymodel.nr_bodies == 1) - DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_NR_SIGN) = (RFLOAT) my_nr_significant_coarse_samples; + // Store nr_significant_coarse_samples for this particle + // Don't do this for multibody, as it would be overwritten for each body, + // and we also use METADATA_NR_SIGN in the new safeguard for the gold-standard separation + if (baseMLO->mymodel.nr_bodies == 1) + DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_NR_SIGN) = (RFLOAT) my_nr_significant_coarse_samples; - AccPtr Mcoarse_significant = ptrFactory.make(ipart_length); - Mcoarse_significant.setHostPtr(&op.Mcoarse_significant.data[offset]); + AccPtr Mcoarse_significant = ptrFactory.make(ipart_length); + Mcoarse_significant.setHostPtr(&op.Mcoarse_significant.data[offset]); - CUSTOM_ALLOCATOR_REGION_NAME("CASDTW_SIG"); - Mcoarse_significant.deviceAlloc(); + CUSTOM_ALLOCATOR_REGION_NAME("CASDTW_SIG"); + Mcoarse_significant.deviceAlloc(); - DEBUG_HANDLE_ERROR(cudaStreamSynchronize(cudaStreamPerThread)); - arrayOverThreshold(unsorted_ipart, Mcoarse_significant, significant_weight); - Mcoarse_significant.cpToHost(); - DEBUG_HANDLE_ERROR(cudaStreamSynchronize(cudaStreamPerThread)); - } - else if (ipart_length == 1) - { - op.Mcoarse_significant.data[img_id * op.Mweight.xdim + sp.nr_dir * sp.nr_psi * sp.nr_trans * sp.iclass_min] = 1; - } - else - CRITICAL(ERRNEGLENGTH); - } - else - { + DEBUG_HANDLE_ERROR(cudaStreamSynchronize(cudaStreamPerThread)); + arrayOverThreshold(unsorted_ipart, Mcoarse_significant, significant_weight); + Mcoarse_significant.cpToHost(); + DEBUG_HANDLE_ERROR(cudaStreamSynchronize(cudaStreamPerThread)); + } + else if (ipart_length == 1) + { + op.Mcoarse_significant.data[sp.nr_dir * sp.nr_psi * sp.nr_trans * sp.iclass_min] = 1; + } + else + CRITICAL(ERRNEGLENGTH); + } + else + { - for (int exp_iclass = sp.iclass_min; exp_iclass <= sp.iclass_max; exp_iclass++) - DEBUG_HANDLE_ERROR(cudaStreamSynchronize(accMLO->classStreams[exp_iclass])); - DEBUG_HANDLE_ERROR(cudaStreamSynchronize(cudaStreamPerThread)); + for (int exp_iclass = sp.iclass_min; exp_iclass <= sp.iclass_max; exp_iclass++) + DEBUG_HANDLE_ERROR(cudaStreamSynchronize(accMLO->classStreams[exp_iclass])); + DEBUG_HANDLE_ERROR(cudaStreamSynchronize(cudaStreamPerThread)); - XFLOAT weights_max = -std::numeric_limits::max(); + XFLOAT weights_max = -std::numeric_limits::max(); - pdf_offset.streamSync(); + pdf_offset.streamSync(); - for (unsigned long exp_iclass = sp.iclass_min; exp_iclass <= sp.iclass_max; exp_iclass++) // TODO could use classStreams - { - if ((baseMLO->mymodel.pdf_class[exp_iclass] > 0.) && (FPCMasks[img_id][exp_iclass].weightNum > 0) ) - { - // Use the constructed mask to build a partial (class-specific) input - // (until now, PassWeights has been an empty placeholder. We now create class-partials pointing at it, and start to fill it with stuff) - - IndexedDataArray thisClassPassWeights(PassWeights[img_id],FPCMasks[img_id][exp_iclass]); - - AccPtr pdf_orientation_class = ptrFactory.make(sp.nr_dir*sp.nr_psi), - pdf_offset_class = ptrFactory.make(sp.nr_trans); - AccPtr pdf_orientation_zeros_class = ptrFactory.make(sp.nr_dir*sp.nr_psi), - pdf_offset_zeros_class = ptrFactory.make(sp.nr_trans); - - pdf_orientation_class .setAccPtr(&((~pdf_orientation) [(exp_iclass-sp.iclass_min)*sp.nr_dir*sp.nr_psi])); - pdf_orientation_zeros_class.setAccPtr(&((~pdf_orientation_zeros)[(exp_iclass-sp.iclass_min)*sp.nr_dir*sp.nr_psi])); - - pdf_offset_class .setAccPtr(&((~pdf_offset) [(exp_iclass-sp.iclass_min)*sp.nr_trans])); - pdf_offset_zeros_class .setAccPtr(&((~pdf_offset_zeros) [(exp_iclass-sp.iclass_min)*sp.nr_trans])); - - thisClassPassWeights.weights.setStream(accMLO->classStreams[exp_iclass]); - - AccUtilities::kernel_exponentiate_weights_fine( - ~pdf_orientation_class, - ~pdf_orientation_zeros_class, - ~pdf_offset_class, - ~pdf_offset_zeros_class, - ~thisClassPassWeights.weights, - (XFLOAT)op.min_diff2[img_id], - sp.nr_oversampled_rot, - sp.nr_oversampled_trans, - ~thisClassPassWeights.rot_id, - ~thisClassPassWeights.trans_idx, - ~FPCMasks[img_id][exp_iclass].jobOrigin, - ~FPCMasks[img_id][exp_iclass].jobExtent, - FPCMasks[img_id][exp_iclass].jobNum, - accMLO->classStreams[exp_iclass]); - - XFLOAT m = AccUtilities::getMaxOnDevice(thisClassPassWeights.weights); - - if (m > weights_max) - weights_max = m; - } - } + for (unsigned long exp_iclass = sp.iclass_min; exp_iclass <= sp.iclass_max; exp_iclass++) // TODO could use classStreams + { + if ((baseMLO->mymodel.pdf_class[exp_iclass] > 0.) && (FPCMasks[exp_iclass].weightNum > 0) ) + { + // Use the constructed mask to build a partial (class-specific) input + // (until now, PassWeights has been an empty placeholder. We now create class-partials pointing at it, and start to fill it with stuff) + + IndexedDataArray thisClassPassWeights(PassWeights,FPCMasks[exp_iclass]); + + AccPtr pdf_orientation_class = ptrFactory.make(sp.nr_dir*sp.nr_psi), + pdf_offset_class = ptrFactory.make(sp.nr_trans); + AccPtr pdf_orientation_zeros_class = ptrFactory.make(sp.nr_dir*sp.nr_psi), + pdf_offset_zeros_class = ptrFactory.make(sp.nr_trans); + + pdf_orientation_class .setAccPtr(&((~pdf_orientation) [(exp_iclass-sp.iclass_min)*sp.nr_dir*sp.nr_psi])); + pdf_orientation_zeros_class.setAccPtr(&((~pdf_orientation_zeros)[(exp_iclass-sp.iclass_min)*sp.nr_dir*sp.nr_psi])); + + pdf_offset_class .setAccPtr(&((~pdf_offset) [(exp_iclass-sp.iclass_min)*sp.nr_trans])); + pdf_offset_zeros_class .setAccPtr(&((~pdf_offset_zeros) [(exp_iclass-sp.iclass_min)*sp.nr_trans])); + + thisClassPassWeights.weights.setStream(accMLO->classStreams[exp_iclass]); + + AccUtilities::kernel_exponentiate_weights_fine( + ~pdf_orientation_class, + ~pdf_orientation_zeros_class, + ~pdf_offset_class, + ~pdf_offset_zeros_class, + ~thisClassPassWeights.weights, + (XFLOAT)op.min_diff2, + sp.nr_oversampled_rot, + sp.nr_oversampled_trans, + ~thisClassPassWeights.rot_id, + ~thisClassPassWeights.trans_idx, + ~FPCMasks[exp_iclass].jobOrigin, + ~FPCMasks[exp_iclass].jobExtent, + FPCMasks[exp_iclass].jobNum, + accMLO->classStreams[exp_iclass]); + + XFLOAT m = AccUtilities::getMaxOnDevice(thisClassPassWeights.weights); + + if (m > weights_max) + weights_max = m; + } + } - for (unsigned long exp_iclass = sp.iclass_min; exp_iclass <= sp.iclass_max; exp_iclass++) // TODO could use classStreams - { - if ((baseMLO->mymodel.pdf_class[exp_iclass] > 0.) && (FPCMasks[img_id][exp_iclass].weightNum > 0) ) - { - IndexedDataArray thisClassPassWeights(PassWeights[img_id],FPCMasks[img_id][exp_iclass]); - - thisClassPassWeights.weights.setStream(accMLO->classStreams[exp_iclass]); - /* - * Add 50 since we want to stay away from e^88, which approaches the single precision limit. - * We still want as high numbers as possible to utilize most of the single precision span. - * Dari - 201710 - */ - AccUtilities::kernel_exponentiate( thisClassPassWeights.weights, 50 - weights_max ); - } - } + for (unsigned long exp_iclass = sp.iclass_min; exp_iclass <= sp.iclass_max; exp_iclass++) // TODO could use classStreams + { + if ((baseMLO->mymodel.pdf_class[exp_iclass] > 0.) && (FPCMasks[exp_iclass].weightNum > 0) ) + { + IndexedDataArray thisClassPassWeights(PassWeights,FPCMasks[exp_iclass]); + + thisClassPassWeights.weights.setStream(accMLO->classStreams[exp_iclass]); + /* + * Add 50 since we want to stay away from e^88, which approaches the single precision limit. + * We still want as high numbers as possible to utilize most of the single precision span. + * Dari - 201710 + */ + AccUtilities::kernel_exponentiate( thisClassPassWeights.weights, 50 - weights_max ); + } + } - op.min_diff2[img_id] += 50 - weights_max; + op.min_diff2 += 50 - weights_max; - for (unsigned long exp_iclass = sp.iclass_min; exp_iclass <= sp.iclass_max; exp_iclass++) - DEBUG_HANDLE_ERROR(cudaStreamSynchronize(accMLO->classStreams[exp_iclass])); - DEBUG_HANDLE_ERROR(cudaStreamSynchronize(cudaStreamPerThread)); + for (unsigned long exp_iclass = sp.iclass_min; exp_iclass <= sp.iclass_max; exp_iclass++) + DEBUG_HANDLE_ERROR(cudaStreamSynchronize(accMLO->classStreams[exp_iclass])); + DEBUG_HANDLE_ERROR(cudaStreamSynchronize(cudaStreamPerThread)); - if(baseMLO->is_som_iter) { - op.sum_weight_class[img_id].resize(baseMLO->mymodel.nr_classes, 0); + if(baseMLO->is_som_iter) { + op.sum_weight_class.resize(baseMLO->mymodel.nr_classes, 0); - for (unsigned long exp_iclass = sp.iclass_min; - exp_iclass <= sp.iclass_max; exp_iclass++) // TODO could use classStreams - { - if ((baseMLO->mymodel.pdf_class[exp_iclass] > 0.) && - (FPCMasks[img_id][exp_iclass].weightNum > 0)) { - IndexedDataArray thisClassPassWeights(PassWeights[img_id], FPCMasks[img_id][exp_iclass]); - op.sum_weight_class[img_id][exp_iclass] = AccUtilities::getSumOnDevice(thisClassPassWeights.weights); - } - } - } + for (unsigned long exp_iclass = sp.iclass_min; + exp_iclass <= sp.iclass_max; exp_iclass++) // TODO could use classStreams + { + if ((baseMLO->mymodel.pdf_class[exp_iclass] > 0.) && + (FPCMasks[exp_iclass].weightNum > 0)) { + IndexedDataArray thisClassPassWeights(PassWeights, FPCMasks[exp_iclass]); + op.sum_weight_class[exp_iclass] = AccUtilities::getSumOnDevice(thisClassPassWeights.weights); + } + } + } - PassWeights[img_id].weights.cpToHost(); // note that the host-pointer is shared: we're copying to Mweight. + PassWeights.weights.cpToHost(); // note that the host-pointer is shared: we're copying to Mweight. - CTIC(accMLO->timer,"sort"); - DEBUG_HANDLE_ERROR(cudaStreamSynchronize(cudaStreamPerThread)); - size_t weightSize = PassWeights[img_id].weights.getSize(); + CTIC(accMLO->timer,"sort"); + DEBUG_HANDLE_ERROR(cudaStreamSynchronize(cudaStreamPerThread)); + size_t weightSize = PassWeights.weights.getSize(); - AccPtr sorted = ptrFactory.make((size_t)weightSize); - AccPtr cumulative_sum = ptrFactory.make((size_t)weightSize); + AccPtr sorted = ptrFactory.make((size_t)weightSize); + AccPtr cumulative_sum = ptrFactory.make((size_t)weightSize); - CUSTOM_ALLOCATOR_REGION_NAME("CASDTW_FINE"); + CUSTOM_ALLOCATOR_REGION_NAME("CASDTW_FINE"); - sorted.accAlloc(); - cumulative_sum.accAlloc(); + sorted.accAlloc(); + cumulative_sum.accAlloc(); - AccUtilities::sortOnDevice(PassWeights[img_id].weights, sorted); - AccUtilities::scanOnDevice(sorted, cumulative_sum); - CTOC(accMLO->timer,"sort"); + AccUtilities::sortOnDevice(PassWeights.weights, sorted); + AccUtilities::scanOnDevice(sorted, cumulative_sum); + CTOC(accMLO->timer,"sort"); - if(baseMLO->adaptive_oversampling!=0) - { - op.sum_weight[img_id] = cumulative_sum.getAccValueAt(cumulative_sum.getSize() - 1); + if(baseMLO->adaptive_oversampling!=0) + { + op.sum_weight = cumulative_sum.getAccValueAt(cumulative_sum.getSize() - 1); - if (op.sum_weight[img_id]==0) - { - std::cerr << std::endl; - std::cerr << " fn_img= " << sp.current_img << std::endl; - std::cerr << " op.part_id= " << op.part_id << std::endl; - std::cerr << " img_id= " << img_id << std::endl; - std::cerr << " op.min_diff2[img_id]= " << op.min_diff2[img_id] << std::endl; - int group_id = baseMLO->mydata.getGroupId(op.part_id, img_id); - std::cerr << " group_id= " << group_id << std::endl; - int optics_group = baseMLO->mydata.getOpticsGroup(op.part_id, img_id); - std::cerr << " optics_group= " << optics_group << std::endl; - std::cerr << " ml_model.scale_correction[group_id]= " << baseMLO->mymodel.scale_correction[group_id] << std::endl; - std::cerr << " exp_significant_weight[img_id]= " << op.significant_weight[img_id] << std::endl; - std::cerr << " exp_max_weight[img_id]= " << op.max_weight[img_id] << std::endl; - std::cerr << " ml_model.sigma2_noise[optics_group]= " << baseMLO->mymodel.sigma2_noise[optics_group] << std::endl; - CRITICAL(ERRSUMWEIGHTZERO); //"op.sum_weight[img_id]==0" - } + if (op.sum_weight==0) + { + std::cerr << std::endl; + std::cerr << " fn_img= " << sp.current_img << std::endl; + std::cerr << " op.part_id= " << op.part_id << std::endl; + std::cerr << " op.min_diff2= " << op.min_diff2 << std::endl; + int group_id = baseMLO->mydata.getGroupId(op.part_id, 0); + std::cerr << " group_id= " << group_id << std::endl; + int optics_group = baseMLO->mydata.getOpticsGroup(op.part_id, 0); + std::cerr << " optics_group= " << optics_group << std::endl; + std::cerr << " ml_model.scale_correction[group_id]= " << baseMLO->mymodel.scale_correction[group_id] << std::endl; + std::cerr << " exp_significant_weight= " << op.significant_weight << std::endl; + std::cerr << " exp_max_weight= " << op.max_weight << std::endl; + std::cerr << " ml_model.sigma2_noise[optics_group]= " << baseMLO->mymodel.sigma2_noise[optics_group] << std::endl; + CRITICAL(ERRSUMWEIGHTZERO); //"op.sum_weight[img_id]==0" + } - size_t thresholdIdx = findThresholdIdxInCumulativeSum(cumulative_sum, (1 - baseMLO->adaptive_fraction) * op.sum_weight[img_id]); - my_significant_weight = sorted.getAccValueAt(thresholdIdx); + size_t thresholdIdx = findThresholdIdxInCumulativeSum(cumulative_sum, (1 - baseMLO->adaptive_fraction) * op.sum_weight); + my_significant_weight = sorted.getAccValueAt(thresholdIdx); - CTIC(accMLO->timer,"getArgMaxOnDevice"); - std::pair max_pair = AccUtilities::getArgMaxOnDevice(PassWeights[img_id].weights); - CTOC(accMLO->timer,"getArgMaxOnDevice"); - op.max_index[img_id].fineIdx = PassWeights[img_id].ihidden_overs[max_pair.first]; - op.max_weight[img_id] = max_pair.second; - } - else - { - my_significant_weight = sorted.getAccValueAt(0); - } - } - CTOC(accMLO->timer,"sumweight1"); - } + CTIC(accMLO->timer,"getArgMaxOnDevice"); + std::pair max_pair = AccUtilities::getArgMaxOnDevice(PassWeights.weights); + CTOC(accMLO->timer,"getArgMaxOnDevice"); + op.max_index.fineIdx = PassWeights.ihidden_overs[max_pair.first]; + op.max_weight = max_pair.second; + } + else + { + my_significant_weight = sorted.getAccValueAt(0); + } + } + CTOC(accMLO->timer,"sumweight1"); + } - op.significant_weight[img_id] = (RFLOAT) my_significant_weight; - } // end loop img_id + op.significant_weight = (RFLOAT) my_significant_weight; #ifdef TIMING diff --git a/src/acc/cpu/cpu_kernels/diff2.h b/src/acc/cpu/cpu_kernels/diff2.h index cd6d0863d..d6fa404a5 100644 --- a/src/acc/cpu/cpu_kernels/diff2.h +++ b/src/acc/cpu/cpu_kernels/diff2.h @@ -741,7 +741,7 @@ inline void diff2_fine_2D( for (unsigned long itrans=0; itrans= XSIZE(exp_Mcoarse_significant)) - { - std::cerr << " ihidden= " << ihidden << " XSIZE(exp_Mcoarse_significant)= " << XSIZE(exp_Mcoarse_significant) << std::endl; - std::cerr << " iorient= " << iorient << " itrans= " << itrans << " exp_nr_trans= " << exp_nr_trans << std::endl; - REPORT_ERROR("ihidden > XSIZE: "); - } + if (ihidden >= XSIZE(exp_Mcoarse_significant)) + { + std::cerr << " ihidden= " << ihidden << " XSIZE(exp_Mcoarse_significant)= " << XSIZE(exp_Mcoarse_significant) << std::endl; + std::cerr << " iorient= " << iorient << " itrans= " << itrans << " exp_nr_trans= " << exp_nr_trans << std::endl; + REPORT_ERROR("ihidden > XSIZE: "); + } #endif - if (DIRECT_A2D_ELEM(exp_Mcoarse_significant, ipart, ihidden)) - return true; - } - } + if (DIRECT_A1D_ELEM(exp_Mcoarse_significant, ihidden)) + return true; + } return false; } @@ -6742,7 +6739,7 @@ void MlOptimiser::getAllSquaredDifferences(long int part_id, int ibody, int exp_ipass, int exp_current_oversampling, int metadata_offset, int exp_idir_min, int exp_idir_max, int exp_ipsi_min, int exp_ipsi_max, int exp_itrans_min, int exp_itrans_max, int exp_iclass_min, int exp_iclass_max, - std::vector &exp_min_diff2, + RFLOAT &exp_min_diff2, std::vector &exp_highres_Xi2_img, std::vector > &exp_Fimg, std::vector > &exp_Fctf, @@ -6783,7 +6780,7 @@ void MlOptimiser::getAllSquaredDifferences(long int part_id, int ibody, long int exp_nr_oversampled_rot = sampling.oversamplingFactorOrientations(exp_current_oversampling); long int exp_nr_oversampled_trans = sampling.oversamplingFactorTranslations(exp_current_oversampling); - exp_Mweight.resize(exp_nr_images, mymodel.nr_classes * exp_nr_dir * exp_nr_psi * exp_nr_trans * exp_nr_oversampled_rot * exp_nr_oversampled_trans); + exp_Mweight.resize(mymodel.nr_classes * exp_nr_dir * exp_nr_psi * exp_nr_trans * exp_nr_oversampled_rot * exp_nr_oversampled_trans); exp_Mweight.initConstant(-999.); if (exp_ipass==0) exp_Mcoarse_significant.clear(); @@ -6878,7 +6875,6 @@ void MlOptimiser::getAllSquaredDifferences(long int part_id, int ibody, for (int img_id = 0; img_id < mydata.numberOfImagesInParticle(part_id); img_id++) { - int my_metadata_offset = metadata_offset + img_id; RFLOAT my_pixel_size = mydata.getImagePixelSize(part_id, img_id); int optics_group = mydata.getOpticsGroup(part_id, img_id); @@ -6887,6 +6883,13 @@ void MlOptimiser::getAllSquaredDifferences(long int part_id, int ibody, oversampled_tilt[iover_rot], oversampled_psi[iover_rot], A, false); + Matrix2D Aproj; + if (mydata.is_tomo) + { + Aproj = mydata.getRotationMatrix(part_id, img_id); + REPORT_ERROR("ERROR: TODO: implement rotations of img_id tilt series"); + + } // Project the reference map (into Fref) #ifdef TIMING // Only time one thread, as I also only time one MPI process @@ -6954,10 +6957,12 @@ void MlOptimiser::getAllSquaredDifferences(long int part_id, int ibody, #endif // In the first pass, always proceed // In the second pass, check whether this translations (&orientation) had a significant weight in the first pass - bool do_proceed = (exp_ipass == 0) ? true : DIRECT_A2D_ELEM(exp_Mcoarse_significant, img_id, ihidden); + bool do_proceed = (exp_ipass == 0) ? true : DIRECT_A1D_ELEM(exp_Mcoarse_significant, ihidden); if (do_proceed) { - // Jun01,2015 - Shaoda & Sjors, Helical refinement + if (mydata.is_tomo) REPORT_ERROR("ERROR: TOD: implement what to do with 2D stack translations here!"); + + // Jun01,2015 - Shaoda & Sjors, Helical refinement sampling.getTranslationsInPixel(itrans, exp_current_oversampling, my_pixel_size, oversampled_translations_x, oversampled_translations_y, oversampled_translations_z, (do_helical_refine) && (!ignore_helical_symmetry)); for (long int iover_trans = 0; iover_trans < exp_nr_oversampled_trans; iover_trans++) @@ -7002,9 +7007,11 @@ void MlOptimiser::getAllSquaredDifferences(long int part_id, int ibody, if (mymodel.data_dim == 3) zshift = (exp_current_oversampling == 0) ? (oversampled_translations_z[0]) : (oversampled_translations_z[iover_trans]); - RFLOAT rot_deg = DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_ROT); - RFLOAT tilt_deg = DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_TILT); - RFLOAT psi_deg = DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_PSI); + if (mydata.is_tomo) REPORT_ERROR("ERROR; TODO: think about the below for 2D shifts in stacks..."); + + RFLOAT rot_deg = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ROT); + RFLOAT tilt_deg = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_TILT); + RFLOAT psi_deg = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PSI); transformCartesianAndHelicalCoords( xshift, yshift, zshift, xshift, yshift, zshift, @@ -7014,7 +7021,10 @@ void MlOptimiser::getAllSquaredDifferences(long int part_id, int ibody, use_coarse_size = ((exp_current_oversampling == 0) && (YSIZE(Frefctf) == image_coarse_size[optics_group])) || ((exp_current_oversampling > 0) && (strict_highres_exp > 0.)); - shiftImageInFourierTransformWithTabSincos( + + if (mydata.is_tomo) REPORT_ERROR("ERROR: TOD: implement what to do with 2D stack translations here!"); + + shiftImageInFourierTransformWithTabSincos( exp_local_Fimgs_shifted[img_id][0], Fimg_otfshift, (RFLOAT)mymodel.ori_size, @@ -7282,12 +7292,15 @@ void MlOptimiser::getAllSquaredDifferences(long int part_id, int ibody, REPORT_ERROR("ihidden_over >= XSIZE(Mweight)"); } #endif - DIRECT_A2D_ELEM(exp_Mweight, img_id, ihidden_over) = diff2; + + + // SHWS 6July2022: += instead of =, as summing over all imag_id.... + DIRECT_A1D_ELEM(exp_Mweight, ihidden_over) += diff2; // Keep track of minimum of all diff2, only for the last image in this series - if (diff2 < exp_min_diff2[img_id]) + if (img_id == mydata.numberOfImagesInParticle(part_id) -1 && diff2 < exp_min_diff2) { - exp_min_diff2[img_id] = diff2; + exp_min_diff2 = diff2; /* if (part_id == 0) { @@ -7328,8 +7341,8 @@ void MlOptimiser::convertAllSquaredDifferencesToWeights(long int part_id, int ib int exp_idir_min, int exp_idir_max, int exp_ipsi_min, int exp_ipsi_max, int exp_itrans_min, int exp_itrans_max, int exp_iclass_min, int exp_iclass_max, MultidimArray &exp_Mweight, MultidimArray &exp_Mcoarse_significant, - std::vector &exp_significant_weight, std::vector &exp_sum_weight, - std::vector > &exp_old_offset, std::vector > &exp_prior, std::vector &exp_min_diff2, + RFLOAT &exp_significant_weight, RFLOAT &exp_sum_weight, + Matrix1D &exp_old_offset, Matrix1D &exp_prior, RFLOAT &exp_min_diff2, std::vector &exp_pointer_dir_nonzeroprior, std::vector &exp_pointer_psi_nonzeroprior, std::vector &exp_directions_prior, std::vector &exp_psi_prior) { @@ -7353,8 +7366,7 @@ void MlOptimiser::convertAllSquaredDifferencesToWeights(long int part_id, int ib long int exp_nr_oversampled_trans = sampling.oversamplingFactorTranslations(exp_current_oversampling); // Initialising... - exp_sum_weight.clear(); - exp_sum_weight.resize(exp_nr_images, 0.); + exp_sum_weight = 0.; RFLOAT my_sigma2_offset = (mymodel.nr_bodies > 1) ? mymodel.sigma_offset_bodies[ibody]*mymodel.sigma_offset_bodies[ibody] : mymodel.sigma2_offset; @@ -7367,481 +7379,465 @@ void MlOptimiser::convertAllSquaredDifferencesToWeights(long int part_id, int ib long int opt_ihidden, opt_ihidden_over; #endif - // loop over all images inside this particle - for (int img_id = 0; img_id < exp_nr_images; img_id++) - { - int my_metadata_offset = metadata_offset + img_id; - RFLOAT my_pixel_size = mydata.getImagePixelSize(part_id, img_id); + RFLOAT my_pixel_size = mydata.getImagePixelSize(part_id, 0); - RFLOAT exp_thisimage_sumweight = 0.; - RFLOAT old_offset_x, old_offset_y, old_offset_z; - if (mymodel.nr_bodies > 1) - { - old_offset_x = old_offset_y = old_offset_z = 0.; - } - else - { - old_offset_x = XX(exp_old_offset[img_id]); - old_offset_y = YY(exp_old_offset[img_id]); - if (mymodel.data_dim == 3) - old_offset_z = ZZ(exp_old_offset[img_id]); - } + RFLOAT exp_thisimage_sumweight = 0.; + RFLOAT old_offset_x, old_offset_y, old_offset_z; + if (mymodel.nr_bodies > 1) + { + old_offset_x = old_offset_y = old_offset_z = 0.; + } + else + { + old_offset_x = XX(exp_old_offset); + old_offset_y = YY(exp_old_offset); + if (mymodel.data_dim == 3) + old_offset_z = ZZ(exp_old_offset); + } - if ((iter == 1 && do_firstiter_cc) || do_always_cc) - { - // Binarize the squared differences array to skip marginalisation - RFLOAT mymindiff2 = 99.e10; - long int myminidx = -1; - // Find the smallest element in this row of exp_Mweight - for (long int i = 0; i < XSIZE(exp_Mweight); i++) - { + if ((iter == 1 && do_firstiter_cc) || do_always_cc) + { + // Binarize the squared differences array to skip marginalisation + RFLOAT mymindiff2 = 99.e10; + long int myminidx = -1; + // Find the smallest element in this row of exp_Mweight + for (long int i = 0; i < XSIZE(exp_Mweight); i++) + { - RFLOAT cc = DIRECT_A2D_ELEM(exp_Mweight, img_id, i); - // ignore non-determined cc - if (cc == -999.) - continue; + RFLOAT cc = DIRECT_A1D_ELEM(exp_Mweight, i); + // ignore non-determined cc + if (cc == -999.) + continue; - // just search for the maximum - if (cc < mymindiff2) - { - mymindiff2 = cc; - myminidx = i; - } - } - // Set all except for the best hidden variable to zero and the smallest element to 1 - for (long int i = 0; i < XSIZE(exp_Mweight); i++) - DIRECT_A2D_ELEM(exp_Mweight, img_id, i)= 0.; + // just search for the maximum + if (cc < mymindiff2) + { + mymindiff2 = cc; + myminidx = i; + } + } + // Set all except for the best hidden variable to zero and the smallest element to 1 + for (long int i = 0; i < XSIZE(exp_Mweight); i++) + DIRECT_A1D_ELEM(exp_Mweight, i)= 0.; - DIRECT_A2D_ELEM(exp_Mweight, img_id, myminidx)= 1.; - exp_thisimage_sumweight += 1.; + DIRECT_A1D_ELEM(exp_Mweight, myminidx)= 1.; + exp_thisimage_sumweight += 1.; - } - else - { - // Extra normalization - RFLOAT pdf_orientation_mean(0),pdf_offset_mean(0); - unsigned long pdf_orientation_count(0), pdf_offset_count(0); - for (int exp_iclass = exp_iclass_min; exp_iclass <= exp_iclass_max; exp_iclass++) - { - for (long int idir = exp_idir_min, iorient = 0; idir <= exp_idir_max; idir++) - for (long int ipsi = exp_ipsi_min; ipsi <= exp_ipsi_max; ipsi++, iorient++) - { - if (do_skip_align || do_skip_rotate) - pdf_orientation_mean += mymodel.pdf_class[exp_iclass]; - else if (mymodel.orientational_prior_mode == NOPRIOR) - pdf_orientation_mean += DIRECT_MULTIDIM_ELEM(mymodel.pdf_direction[exp_iclass], idir); - else - pdf_orientation_mean += exp_directions_prior[idir] * exp_psi_prior[ipsi]; - pdf_orientation_count++; - } + } + else + { + // Extra normalization + RFLOAT pdf_orientation_mean(0),pdf_offset_mean(0); + unsigned long pdf_orientation_count(0), pdf_offset_count(0); + for (int exp_iclass = exp_iclass_min; exp_iclass <= exp_iclass_max; exp_iclass++) + { + for (long int idir = exp_idir_min, iorient = 0; idir <= exp_idir_max; idir++) + for (long int ipsi = exp_ipsi_min; ipsi <= exp_ipsi_max; ipsi++, iorient++) + { + if (do_skip_align || do_skip_rotate) + pdf_orientation_mean += mymodel.pdf_class[exp_iclass]; + else if (mymodel.orientational_prior_mode == NOPRIOR) + pdf_orientation_mean += DIRECT_MULTIDIM_ELEM(mymodel.pdf_direction[exp_iclass], idir); + else + pdf_orientation_mean += exp_directions_prior[idir] * exp_psi_prior[ipsi]; + pdf_orientation_count++; + } - RFLOAT myprior_x, myprior_y, myprior_z; - if (mymodel.nr_bodies > 1) - { - myprior_x = myprior_y = myprior_z = 0.; - } - else if (mymodel.ref_dim == 2 && !do_helical_refine) - { - myprior_x = XX(mymodel.prior_offset_class[exp_iclass]); - myprior_y = YY(mymodel.prior_offset_class[exp_iclass]); - } - else - { - myprior_x = XX(exp_prior[img_id]); - myprior_y = YY(exp_prior[img_id]); - if (mymodel.data_dim == 3) - myprior_z = ZZ(exp_prior[img_id]); - } - for (long int itrans = exp_itrans_min; itrans <= exp_itrans_max; itrans++) - { - RFLOAT offset_x = old_offset_x + sampling.translations_x[itrans]; - RFLOAT offset_y = old_offset_y + sampling.translations_y[itrans]; - RFLOAT tdiff2 = 0.; - if ( (!do_helical_refine) || (ignore_helical_symmetry) || (mymodel.data_dim == 3) ) - tdiff2 += (offset_x - myprior_x) * (offset_x - myprior_x); - tdiff2 += (offset_y - myprior_y) * (offset_y - myprior_y); - if (mymodel.data_dim == 3) - { - RFLOAT offset_z = old_offset_z + sampling.translations_z[itrans]; - if ( (!do_helical_refine) || (ignore_helical_symmetry) ) - tdiff2 += (offset_z - myprior_z) * (offset_z - myprior_z); - } - // As of version 3.1, sigma_offsets are in Angstroms! - tdiff2 *= my_pixel_size * my_pixel_size; - - // P(offset|sigma2_offset) - // This is the probability of the offset, given the model offset and variance. - RFLOAT pdf_offset; - if (my_sigma2_offset < 0.0001) - pdf_offset_mean += ( tdiff2 > 0.) ? 0. : 1.; - else - pdf_offset_mean += exp ( tdiff2 / (-2. * my_sigma2_offset) ) / ( 2. * PI * my_sigma2_offset ); - pdf_offset_count ++; - } - } - pdf_orientation_mean /= (RFLOAT) pdf_orientation_count; - pdf_offset_mean /= (RFLOAT) pdf_offset_count; - // Loop from iclass_min to iclass_max to deal with seed generation in first iteration - for (int exp_iclass = exp_iclass_min; exp_iclass <= exp_iclass_max; exp_iclass++) - { + RFLOAT myprior_x, myprior_y, myprior_z; + if (mymodel.nr_bodies > 1) + { + myprior_x = myprior_y = myprior_z = 0.; + } + else if (mymodel.ref_dim == 2 && !do_helical_refine) + { + myprior_x = XX(mymodel.prior_offset_class[exp_iclass]); + myprior_y = YY(mymodel.prior_offset_class[exp_iclass]); + } + else + { + myprior_x = XX(exp_prior); + myprior_y = YY(exp_prior); + if (mymodel.data_dim == 3) + myprior_z = ZZ(exp_prior); + } + for (long int itrans = exp_itrans_min; itrans <= exp_itrans_max; itrans++) + { + RFLOAT offset_x = old_offset_x + sampling.translations_x[itrans]; + RFLOAT offset_y = old_offset_y + sampling.translations_y[itrans]; + RFLOAT tdiff2 = 0.; + if ( (!do_helical_refine) || (ignore_helical_symmetry) || (mymodel.data_dim == 3) ) + tdiff2 += (offset_x - myprior_x) * (offset_x - myprior_x); + tdiff2 += (offset_y - myprior_y) * (offset_y - myprior_y); + if (mymodel.data_dim == 3) + { + RFLOAT offset_z = old_offset_z + sampling.translations_z[itrans]; + if ( (!do_helical_refine) || (ignore_helical_symmetry) ) + tdiff2 += (offset_z - myprior_z) * (offset_z - myprior_z); + } + // As of version 3.1, sigma_offsets are in Angstroms! + tdiff2 *= my_pixel_size * my_pixel_size; + + // P(offset|sigma2_offset) + // This is the probability of the offset, given the model offset and variance. + RFLOAT pdf_offset; + if (my_sigma2_offset < 0.0001) + pdf_offset_mean += ( tdiff2 > 0.) ? 0. : 1.; + else + pdf_offset_mean += exp ( tdiff2 / (-2. * my_sigma2_offset) ) / ( 2. * PI * my_sigma2_offset ); + pdf_offset_count ++; + } + } + pdf_orientation_mean /= (RFLOAT) pdf_orientation_count; + pdf_offset_mean /= (RFLOAT) pdf_offset_count; + // Loop from iclass_min to iclass_max to deal with seed generation in first iteration + for (int exp_iclass = exp_iclass_min; exp_iclass <= exp_iclass_max; exp_iclass++) + { - // Make PdfOffset calculation much faster... - RFLOAT myprior_x, myprior_y, myprior_z; - if (mymodel.nr_bodies > 1) - { - myprior_x = myprior_y = myprior_z = 0.; - } - else if (mymodel.ref_dim == 2) - { - myprior_x = XX(mymodel.prior_offset_class[exp_iclass]); - myprior_y = YY(mymodel.prior_offset_class[exp_iclass]); - } - else - { - myprior_x = XX(exp_prior[img_id]); - myprior_y = YY(exp_prior[img_id]); - if (mymodel.data_dim == 3) - myprior_z = ZZ(exp_prior[img_id]); - } - for (long int idir = exp_idir_min, iorient = 0; idir <= exp_idir_max; idir++) - { - for (long int ipsi = exp_ipsi_min; ipsi <= exp_ipsi_max; ipsi++, iorient++) - { - long int iorientclass = exp_iclass * exp_nr_dir * exp_nr_psi + iorient; - RFLOAT pdf_orientation; + // Make PdfOffset calculation much faster... + RFLOAT myprior_x, myprior_y, myprior_z; + if (mymodel.nr_bodies > 1) + { + myprior_x = myprior_y = myprior_z = 0.; + } + else if (mymodel.ref_dim == 2) + { + myprior_x = XX(mymodel.prior_offset_class[exp_iclass]); + myprior_y = YY(mymodel.prior_offset_class[exp_iclass]); + } + else + { + myprior_x = XX(exp_prior); + myprior_y = YY(exp_prior); + if (mymodel.data_dim == 3) + myprior_z = ZZ(exp_prior); + } + for (long int idir = exp_idir_min, iorient = 0; idir <= exp_idir_max; idir++) + { + for (long int ipsi = exp_ipsi_min; ipsi <= exp_ipsi_max; ipsi++, iorient++) + { + long int iorientclass = exp_iclass * exp_nr_dir * exp_nr_psi + iorient; + RFLOAT pdf_orientation; - // Get prior for this direction - if (do_skip_align || do_skip_rotate) - { - pdf_orientation = mymodel.pdf_class[exp_iclass]; - } - else if (mymodel.orientational_prior_mode == NOPRIOR) - { - pdf_orientation = DIRECT_MULTIDIM_ELEM(mymodel.pdf_direction[exp_iclass], idir); - } - else - { - // P(orientation) = P(idir|dir_prior) * P(ipsi|psi_prior) - // This is the probability of the orientation, given the gathered - // statistics of all assigned orientations of the dataset, since we - // are assigning a gaussian prior to all parameters. - pdf_orientation = exp_directions_prior[idir] * exp_psi_prior[ipsi]; - } + // Get prior for this direction + if (do_skip_align || do_skip_rotate) + { + pdf_orientation = mymodel.pdf_class[exp_iclass]; + } + else if (mymodel.orientational_prior_mode == NOPRIOR) + { + pdf_orientation = DIRECT_MULTIDIM_ELEM(mymodel.pdf_direction[exp_iclass], idir); + } + else + { + // P(orientation) = P(idir|dir_prior) * P(ipsi|psi_prior) + // This is the probability of the orientation, given the gathered + // statistics of all assigned orientations of the dataset, since we + // are assigning a gaussian prior to all parameters. + pdf_orientation = exp_directions_prior[idir] * exp_psi_prior[ipsi]; + } - if (pdf_orientation_mean != 0.) - pdf_orientation /= pdf_orientation_mean; + if (pdf_orientation_mean != 0.) + pdf_orientation /= pdf_orientation_mean; - // Loop over all translations - long int ihidden = iorientclass * exp_nr_trans; - for (long int itrans = exp_itrans_min; itrans <= exp_itrans_max; itrans++, ihidden++) - { - // May18,2015 - Shaoda & Sjors - Helical refinement (translational searches) - // Calculate the vector length of myprior - RFLOAT mypriors_len2 = myprior_x * myprior_x + myprior_y * myprior_y; - if (mymodel.data_dim == 3) - mypriors_len2 += myprior_z * myprior_z; - // If it is doing helical refinement AND Cartesian vector myprior has a length > 0, transform the vector to its helical coordinates - if ( (do_helical_refine) && (!ignore_helical_symmetry) && (mypriors_len2 > 0.00001) ) - { - RFLOAT rot_deg = DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_ROT); - RFLOAT tilt_deg = DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_TILT); - RFLOAT psi_deg = DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_PSI); - transformCartesianAndHelicalCoords(myprior_x, myprior_y, myprior_z, myprior_x, myprior_y, myprior_z, rot_deg, tilt_deg, psi_deg, mymodel.data_dim, CART_TO_HELICAL_COORDS); - } - // (For helical refinement) Now offset, old_offset, sampling.translations and myprior are all in helical coordinates - - // To speed things up, only calculate pdf_offset at the coarse sampling. - // That should not matter much, and that way one does not need to calculate all the OversampledTranslations - RFLOAT offset_x = old_offset_x + sampling.translations_x[itrans]; - RFLOAT offset_y = old_offset_y + sampling.translations_y[itrans]; - RFLOAT tdiff2 = 0.; - if ( (!do_helical_refine) || (ignore_helical_symmetry) || (mymodel.data_dim == 3) ) - tdiff2 += (offset_x - myprior_x) * (offset_x - myprior_x); - tdiff2 += (offset_y - myprior_y) * (offset_y - myprior_y); - if (mymodel.data_dim == 3) - { - RFLOAT offset_z = old_offset_z + sampling.translations_z[itrans]; - if ( (!do_helical_refine) || (ignore_helical_symmetry) ) - tdiff2 += (offset_z - myprior_z) * (offset_z - myprior_z); - } + // Loop over all translations + long int ihidden = iorientclass * exp_nr_trans; + for (long int itrans = exp_itrans_min; itrans <= exp_itrans_max; itrans++, ihidden++) + { + // May18,2015 - Shaoda & Sjors - Helical refinement (translational searches) + // Calculate the vector length of myprior + RFLOAT mypriors_len2 = myprior_x * myprior_x + myprior_y * myprior_y; + if (mymodel.data_dim == 3) + mypriors_len2 += myprior_z * myprior_z; + // If it is doing helical refinement AND Cartesian vector myprior has a length > 0, transform the vector to its helical coordinates + if ( (do_helical_refine) && (!ignore_helical_symmetry) && (mypriors_len2 > 0.00001) ) + { + RFLOAT rot_deg = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ROT); + RFLOAT tilt_deg = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_TILT); + RFLOAT psi_deg = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PSI); + transformCartesianAndHelicalCoords(myprior_x, myprior_y, myprior_z, myprior_x, myprior_y, myprior_z, rot_deg, tilt_deg, psi_deg, mymodel.data_dim, CART_TO_HELICAL_COORDS); + } + // (For helical refinement) Now offset, old_offset, sampling.translations and myprior are all in helical coordinates + + // To speed things up, only calculate pdf_offset at the coarse sampling. + // That should not matter much, and that way one does not need to calculate all the OversampledTranslations + RFLOAT offset_x = old_offset_x + sampling.translations_x[itrans]; + RFLOAT offset_y = old_offset_y + sampling.translations_y[itrans]; + RFLOAT tdiff2 = 0.; + if ( (!do_helical_refine) || (ignore_helical_symmetry) || (mymodel.data_dim == 3) ) + tdiff2 += (offset_x - myprior_x) * (offset_x - myprior_x); + tdiff2 += (offset_y - myprior_y) * (offset_y - myprior_y); + if (mymodel.data_dim == 3) + { + RFLOAT offset_z = old_offset_z + sampling.translations_z[itrans]; + if ( (!do_helical_refine) || (ignore_helical_symmetry) ) + tdiff2 += (offset_z - myprior_z) * (offset_z - myprior_z); + } - // As of version 3.1, sigma_offsets are in Angstroms! - tdiff2 *= my_pixel_size * my_pixel_size; + // As of version 3.1, sigma_offsets are in Angstroms! + tdiff2 *= my_pixel_size * my_pixel_size; - // P(offset|sigma2_offset) - // This is the probability of the offset, given the model offset and variance. - RFLOAT pdf_offset; - if (my_sigma2_offset < 0.0001) - pdf_offset = ( tdiff2 > 0.) ? 0. : 1.; - else - pdf_offset = exp ( tdiff2 / (-2. * my_sigma2_offset) ) / ( 2. * PI * my_sigma2_offset ); + // P(offset|sigma2_offset) + // This is the probability of the offset, given the model offset and variance. + RFLOAT pdf_offset; + if (my_sigma2_offset < 0.0001) + pdf_offset = ( tdiff2 > 0.) ? 0. : 1.; + else + pdf_offset = exp ( tdiff2 / (-2. * my_sigma2_offset) ) / ( 2. * PI * my_sigma2_offset ); - if (pdf_offset_mean > 0.) - pdf_offset /= pdf_offset_mean; + if (pdf_offset_mean > 0.) + pdf_offset /= pdf_offset_mean; #ifdef TIMING - // Only time one thread, as I also only time one MPI process - if (part_id == mydata.sorted_idx[exp_my_first_part_id]) - timer.tic(TIMING_WEIGHT_EXP); + // Only time one thread, as I also only time one MPI process + if (part_id == mydata.sorted_idx[exp_my_first_part_id]) + timer.tic(TIMING_WEIGHT_EXP); #endif - // Now first loop over iover_rot, because that is the order in exp_Mweight as well - long int ihidden_over = ihidden * exp_nr_oversampled_rot * exp_nr_oversampled_trans; - for (long int iover_rot = 0; iover_rot < exp_nr_oversampled_rot; iover_rot++) - { - // Then loop over iover_trans - for (long int iover_trans = 0; iover_trans < exp_nr_oversampled_trans; iover_trans++, ihidden_over++) - { - // Only exponentiate for determined values of exp_Mweight - // (this is always true in the first pass, but not so in the second pass) - // Only deal with this sampling point if its weight was significant - if (DIRECT_A2D_ELEM(exp_Mweight, img_id, ihidden_over) < 0.) - { - DIRECT_A2D_ELEM(exp_Mweight, img_id, ihidden_over) = 0.; - } - else - { - // Set the weight base to the probability of the parameters given the prior - RFLOAT weight = pdf_orientation * pdf_offset; - RFLOAT diff2 = DIRECT_A2D_ELEM(exp_Mweight, img_id, ihidden_over) - exp_min_diff2[img_id]; - // next line because of numerical precision of exp-function + // Now first loop over iover_rot, because that is the order in exp_Mweight as well + long int ihidden_over = ihidden * exp_nr_oversampled_rot * exp_nr_oversampled_trans; + for (long int iover_rot = 0; iover_rot < exp_nr_oversampled_rot; iover_rot++) + { + // Then loop over iover_trans + for (long int iover_trans = 0; iover_trans < exp_nr_oversampled_trans; iover_trans++, ihidden_over++) + { + // Only exponentiate for determined values of exp_Mweight + // (this is always true in the first pass, but not so in the second pass) + // Only deal with this sampling point if its weight was significant + if (DIRECT_A1D_ELEM(exp_Mweight, ihidden_over) < 0.) + { + DIRECT_A1D_ELEM(exp_Mweight, ihidden_over) = 0.; + } + else + { + // Set the weight base to the probability of the parameters given the prior + RFLOAT weight = pdf_orientation * pdf_offset; + RFLOAT diff2 = DIRECT_A1D_ELEM(exp_Mweight, ihidden_over) - exp_min_diff2; + // next line because of numerical precision of exp-function #ifdef RELION_SINGLE_PRECISION - if (diff2 > 88.) - weight = 0.; + if (diff2 > 88.) + weight = 0.; #else - if (diff2 > 700.) - weight = 0.; + if (diff2 > 700.) + weight = 0.; #endif - // TODO: use tabulated exp function? - else weight *= exp(-diff2); + // TODO: use tabulated exp function? + else weight *= exp(-diff2); - //std::cerr << "ihidden_over= "< 0) - { - FOR_ALL_DIRECT_ELEMENTS_IN_MULTIDIMARRAY(It()) - { - if (DIRECT_MULTIDIM_ELEM(exp_Mcoarse_significant, n)) - DIRECT_MULTIDIM_ELEM(It(), n) = 1.; - else - DIRECT_MULTIDIM_ELEM(It(), n) = 0.; - } - It.write("Mcoarse_significant.spi"); - } - std::cerr << " part_id= " << part_id << std::endl; - std::cerr << " img_id= " << img_id << std::endl; - /* - MultidimArray Faux; - FourierTransformer transformer; - windowFourierTransform(exp_Fimg, Faux, mymodel.ori_size); - It().resize(mymodel.ori_size, mymodel.ori_size); - transformer.inverseFourierTransform(Faux, It()); - CenterFFT(It(), false); - It.write("exp_Fimg.spi"); - std::cerr << "written exp_Fimgs.spi " << std::endl; - */ - int group_id = mydata.getGroupId(part_id, img_id); - int optics_group = mydata.getOpticsGroup(part_id, img_id); - std::cerr << " group_id= " << group_id << " mymodel.scale_correction[group_id]= " << mymodel.scale_correction[group_id] << std::endl; - std::cerr << " exp_ipass= " << exp_ipass << std::endl; - std::cerr << " sampling.NrDirections(0, true)= " << sampling.NrDirections() - << " sampling.NrDirections(0, false)= " << sampling.NrDirections(0, &exp_pointer_dir_nonzeroprior) << std::endl; - std::cerr << " sampling.NrPsiSamplings(0, true)= " << sampling.NrPsiSamplings() - << " sampling.NrPsiSamplings(0, false)= " << sampling.NrPsiSamplings(0, &exp_pointer_psi_nonzeroprior) << std::endl; - std::cerr << " mymodel.sigma2_noise[optics_group]= " << mymodel.sigma2_noise[optics_group] << std::endl; - if (do_norm_correction) - { - std::cerr << " mymodel.avg_norm_correction= " << mymodel.avg_norm_correction << std::endl; - std::cerr << " wsum_model.avg_norm_correction= " << wsum_model.avg_norm_correction << std::endl; - } + if (exp_thisimage_sumweight == 0. || std::isnan(exp_thisimage_sumweight)) + { + std::cerr << " exp_thisimage_sumweight= " << exp_thisimage_sumweight << std::endl; + Image It; + It() = exp_Mweight; + It.write("Mweight.spi"); + //It() = DEBUGGING_COPY_exp_Mweight; + //It.write("Mweight_copy.spi"); + It().resize(exp_Mcoarse_significant); + if (MULTIDIM_SIZE(It()) > 0) + { + FOR_ALL_DIRECT_ELEMENTS_IN_MULTIDIMARRAY(It()) + { + if (DIRECT_MULTIDIM_ELEM(exp_Mcoarse_significant, n)) + DIRECT_MULTIDIM_ELEM(It(), n) = 1.; + else + DIRECT_MULTIDIM_ELEM(It(), n) = 0.; + } + It.write("Mcoarse_significant.spi"); + } + std::cerr << " part_id= " << part_id << std::endl; + /* + MultidimArray Faux; + FourierTransformer transformer; + windowFourierTransform(exp_Fimg, Faux, mymodel.ori_size); + It().resize(mymodel.ori_size, mymodel.ori_size); + transformer.inverseFourierTransform(Faux, It()); + CenterFFT(It(), false); + It.write("exp_Fimg.spi"); + std::cerr << "written exp_Fimgs.spi " << std::endl; + */ + int group_id = mydata.getGroupId(part_id, 0); + int optics_group = mydata.getOpticsGroup(part_id, 0); + std::cerr << " group_id= " << group_id << " mymodel.scale_correction[group_id]= " << mymodel.scale_correction[group_id] << std::endl; + std::cerr << " exp_ipass= " << exp_ipass << std::endl; + std::cerr << " sampling.NrDirections(0, true)= " << sampling.NrDirections() + << " sampling.NrDirections(0, false)= " << sampling.NrDirections(0, &exp_pointer_dir_nonzeroprior) << std::endl; + std::cerr << " sampling.NrPsiSamplings(0, true)= " << sampling.NrPsiSamplings() + << " sampling.NrPsiSamplings(0, false)= " << sampling.NrPsiSamplings(0, &exp_pointer_psi_nonzeroprior) << std::endl; + std::cerr << " mymodel.sigma2_noise[optics_group]= " << mymodel.sigma2_noise[optics_group] << std::endl; + if (do_norm_correction) + { + std::cerr << " mymodel.avg_norm_correction= " << mymodel.avg_norm_correction << std::endl; + std::cerr << " wsum_model.avg_norm_correction= " << wsum_model.avg_norm_correction << std::endl; + } - std::cerr << "written out Mweight.spi" << std::endl; - std::cerr << " exp_thisimage_sumweight= " << exp_thisimage_sumweight << std::endl; - std::cerr << " exp_min_diff2[img_id]= " << exp_min_diff2[img_id] << std::endl; - REPORT_ERROR("ERROR!!! zero sum of weights...."); - } + std::cerr << "written out Mweight.spi" << std::endl; + std::cerr << " exp_thisimage_sumweight= " << exp_thisimage_sumweight << std::endl; + std::cerr << " exp_min_diff2[img_id]= " << exp_min_diff2 << std::endl; + REPORT_ERROR("ERROR!!! zero sum of weights...."); + } #endif - } //end loop img_id - // Initialise exp_Mcoarse_significant if (exp_ipass==0) - exp_Mcoarse_significant.resize(exp_nr_images, XSIZE(exp_Mweight)); + exp_Mcoarse_significant.resize(XSIZE(exp_Mweight)); // Now, for each image, find the exp_significant_weight that encompasses adaptive_fraction of exp_sum_weight - exp_significant_weight.clear(); - exp_significant_weight.resize(exp_nr_images, 0.); - - for (int img_id = 0; img_id < exp_nr_images; img_id++) - { + exp_significant_weight = 0.; - int my_metadata_offset = metadata_offset + img_id; #ifdef TIMING if (part_id == mydata.sorted_idx[exp_my_first_part_id]) timer.tic(TIMING_WEIGHT_SORT); #endif - MultidimArray sorted_weight; - // Get the relevant row for this particle - exp_Mweight.getRow(img_id, sorted_weight); + MultidimArray sorted_weight = exp_Mweight; - // Only select non-zero probabilities to speed up sorting - long int np = 0; - FOR_ALL_DIRECT_ELEMENTS_IN_MULTIDIMARRAY(sorted_weight) - { - if (DIRECT_MULTIDIM_ELEM(sorted_weight, n) > 0.) - { - DIRECT_MULTIDIM_ELEM(sorted_weight, np) = DIRECT_MULTIDIM_ELEM(sorted_weight, n); - np++; - } - } - sorted_weight.resize(np); + // Only select non-zero probabilities to speed up sorting + long int np = 0; + FOR_ALL_DIRECT_ELEMENTS_IN_MULTIDIMARRAY(sorted_weight) + { + if (DIRECT_MULTIDIM_ELEM(sorted_weight, n) > 0.) + { + DIRECT_MULTIDIM_ELEM(sorted_weight, np) = DIRECT_MULTIDIM_ELEM(sorted_weight, n); + np++; + } + } + sorted_weight.resize(np); - // Sort from low to high values - sorted_weight.sort(); + // Sort from low to high values + sorted_weight.sort(); #ifdef TIMING - if (part_id == mydata.sorted_idx[exp_my_first_part_id]) - timer.toc(TIMING_WEIGHT_SORT); + if (part_id == mydata.sorted_idx[exp_my_first_part_id]) + timer.toc(TIMING_WEIGHT_SORT); #endif - RFLOAT frac_weight = 0.; - RFLOAT my_significant_weight; - long int my_nr_significant_coarse_samples = 0; - for (long int i = XSIZE(sorted_weight) - 1; i >= 0; i--) - { - if (maximum_significants > 0 ) - { - if(my_nr_significant_coarse_samples < maximum_significants) - { - if (exp_ipass==0) - my_nr_significant_coarse_samples++; - my_significant_weight = DIRECT_A1D_ELEM(sorted_weight, i); - } - } - else - { - if (exp_ipass==0) - my_nr_significant_coarse_samples++; - my_significant_weight = DIRECT_A1D_ELEM(sorted_weight, i); - } - frac_weight += DIRECT_A1D_ELEM(sorted_weight, i); - if (frac_weight > adaptive_fraction * exp_sum_weight[img_id]) - break; - } + RFLOAT frac_weight = 0.; + RFLOAT my_significant_weight; + long int my_nr_significant_coarse_samples = 0; + for (long int i = XSIZE(sorted_weight) - 1; i >= 0; i--) + { + if (maximum_significants > 0 ) + { + if(my_nr_significant_coarse_samples < maximum_significants) + { + if (exp_ipass==0) + my_nr_significant_coarse_samples++; + my_significant_weight = DIRECT_A1D_ELEM(sorted_weight, i); + } + } + else + { + if (exp_ipass==0) + my_nr_significant_coarse_samples++; + my_significant_weight = DIRECT_A1D_ELEM(sorted_weight, i); + } + frac_weight += DIRECT_A1D_ELEM(sorted_weight, i); + if (frac_weight > adaptive_fraction * exp_sum_weight) + break; + } #ifdef DEBUG_SORT - // Check sorted array is really sorted - RFLOAT prev = 0.; - FOR_ALL_DIRECT_ELEMENTS_IN_MULTIDIMARRAY(sorted_weight) - { - if (DIRECT_MULTIDIM_ELEM(sorted_weight, n) < prev) - { - Image It; - It()=sorted_weight; - It() *= 10000; - It.write("sorted_weight.spi"); - std::cerr << "written sorted_weight.spi" << std::endl; - REPORT_ERROR("Error in sorting!"); - } - prev=DIRECT_MULTIDIM_ELEM(sorted_weight, n); - } -#endif - - if (exp_ipass==0 && my_nr_significant_coarse_samples == 0) - { - std::cerr << " part_id= " << part_id << " img_id= " << img_id << " adaptive_fraction= " << adaptive_fraction << std::endl; - std::cerr << " frac-weight= " << frac_weight << std::endl; - std::cerr << " exp_sum_weight[img_id]= " << exp_sum_weight[img_id] << std::endl; - Image It; - std::cerr << " XSIZE(exp_Mweight)= " << XSIZE(exp_Mweight) << std::endl; - It()=exp_Mweight; - It() *= 10000; - It.write("Mweight2.spi"); - std::cerr << "written Mweight2.spi" << std::endl; - std::cerr << " np= " << np << std::endl; - It()=sorted_weight; - It() *= 10000; - std::cerr << " XSIZE(sorted_weight)= " << XSIZE(sorted_weight) << std::endl; - if (XSIZE(sorted_weight) > 0) - { - It.write("sorted_weight.spi"); - std::cerr << "written sorted_weight.spi" << std::endl; - } - REPORT_ERROR("my_nr_significant_coarse_samples == 0"); - } - - if (exp_ipass==0) - { - // Store nr_significant_coarse_samples for this particle - // Don't do this for multibody, as it would be overwritten for each body, - // and we also use METADATA_NR_SIGN in the new safeguard for the gold-standard separation - if (mymodel.nr_bodies == 1) - DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_NR_SIGN) = (RFLOAT)my_nr_significant_coarse_samples; + // Check sorted array is really sorted + RFLOAT prev = 0.; + FOR_ALL_DIRECT_ELEMENTS_IN_MULTIDIMARRAY(sorted_weight) + { + if (DIRECT_MULTIDIM_ELEM(sorted_weight, n) < prev) + { + Image It; + It()=sorted_weight; + It() *= 10000; + It.write("sorted_weight.spi"); + std::cerr << "written sorted_weight.spi" << std::endl; + REPORT_ERROR("Error in sorting!"); + } + prev=DIRECT_MULTIDIM_ELEM(sorted_weight, n); + } +#endif - // Keep track of which coarse samplings were significant were significant for this particle - for (int ihidden = 0; ihidden < XSIZE(exp_Mcoarse_significant); ihidden++) - { - if (DIRECT_A2D_ELEM(exp_Mweight, img_id, ihidden) >= my_significant_weight) - DIRECT_A2D_ELEM(exp_Mcoarse_significant, img_id, ihidden) = true; - else - DIRECT_A2D_ELEM(exp_Mcoarse_significant, img_id, ihidden) = false; - } + if (exp_ipass==0 && my_nr_significant_coarse_samples == 0) + { + std::cerr << " part_id= " << part_id << " adaptive_fraction= " << adaptive_fraction << std::endl; + std::cerr << " frac-weight= " << frac_weight << std::endl; + std::cerr << " exp_sum_weight= " << exp_sum_weight << std::endl; + Image It; + std::cerr << " XSIZE(exp_Mweight)= " << XSIZE(exp_Mweight) << std::endl; + It()=exp_Mweight; + It() *= 10000; + It.write("Mweight2.spi"); + std::cerr << "written Mweight2.spi" << std::endl; + std::cerr << " np= " << np << std::endl; + It()=sorted_weight; + It() *= 10000; + std::cerr << " XSIZE(sorted_weight)= " << XSIZE(sorted_weight) << std::endl; + if (XSIZE(sorted_weight) > 0) + { + It.write("sorted_weight.spi"); + std::cerr << "written sorted_weight.spi" << std::endl; + } + REPORT_ERROR("my_nr_significant_coarse_samples == 0"); + } - } - exp_significant_weight[img_id] = my_significant_weight; + if (exp_ipass==0) + { + // Store nr_significant_coarse_samples for this particle + // Don't do this for multibody, as it would be overwritten for each body, + // and we also use METADATA_NR_SIGN in the new safeguard for the gold-standard separation + if (mymodel.nr_bodies == 1) + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_NR_SIGN) = (RFLOAT)my_nr_significant_coarse_samples; + + // Keep track of which coarse samplings were significant were significant for this particle + for (int ihidden = 0; ihidden < XSIZE(exp_Mcoarse_significant); ihidden++) + { + if (DIRECT_A1D_ELEM(exp_Mweight, ihidden) >= my_significant_weight) + DIRECT_A1D_ELEM(exp_Mcoarse_significant, ihidden) = true; + else + DIRECT_A1D_ELEM(exp_Mcoarse_significant, ihidden) = false; + } - } // end loop img_id + } + exp_significant_weight = my_significant_weight; #ifdef TIMING if (part_id == mydata.sorted_idx[exp_my_first_part_id]) diff --git a/src/ml_optimiser.h b/src/ml_optimiser.h index 1428b90fb..731273325 100644 --- a/src/ml_optimiser.h +++ b/src/ml_optimiser.h @@ -1097,7 +1097,7 @@ class MlOptimiser int exp_ipass, int exp_current_oversampling, int metadata_offset, int exp_idir_min, int exp_idir_max, int exp_ipsi_min, int exp_ipsi_max, int exp_itrans_min, int exp_itrans_max, int my_iclass_min, int my_iclass_max, - std::vector &exp_min_diff2, + RFLOAT &exp_min_diff2, std::vector &exp_highres_Xi2_img, std::vector > &exp_Fimg, std::vector > &exp_Fctf, @@ -1119,7 +1119,7 @@ class MlOptimiser int exp_itrans_min, int exp_itrans_max, int my_iclass_min, int my_iclass_max, MultidimArray &exp_Mweight, MultidimArray &exp_Mcoarse_significant, std::vector &exp_significant_weight, std::vector &exp_sum_weight, - std::vector > &exp_old_offset, std::vector > &exp_prior, std::vector &exp_min_diff2, + Matrix1D &exp_old_offset, Matrix1D &exp_prior, RFLOAT &exp_min_diff2, std::vector &exp_pointer_dir_nonzeroprior, std::vector &exp_pointer_psi_nonzeroprior, std::vector &exp_directions_prior, std::vector &exp_psi_prior); @@ -1134,13 +1134,13 @@ class MlOptimiser std::vector > &exp_Fimg_nomask, std::vector > &exp_Fctf, std::vector > &exp_power_img, - std::vector > &exp_old_offset, - std::vector > &exp_prior, + Matrix1D &exp_old_offset, + Matrix1D &exp_prior, MultidimArray &exp_Mweight, MultidimArray &exp_Mcoarse_significant, - std::vector &exp_significant_weight, - std::vector &exp_sum_weight, - std::vector &exp_max_weight, + RFLOAT &exp_significant_weight, + RFLOAT &exp_sum_weight, + RFLOAT &exp_max_weight, std::vector &exp_pointer_dir_nonzeroprior, std::vector &exp_pointer_psi_nonzeroprior, std::vector &exp_directions_prior, std::vector &exp_psi_prior, std::vector > > &exp_local_Fimgs_shifted, From 6682ce5d85a999663011f822d1e41317d1247bc8 Mon Sep 17 00:00:00 2001 From: scheres Date: Thu, 7 Jul 2022 09:37:32 +0100 Subject: [PATCH 074/495] continuing on storeWeightedSums, not ready yet --- src/acc/acc_ml_optimiser_impl.h | 300 ++++++++++++++++---------------- src/ml_optimiser.cpp | 17 +- src/ml_optimiser.h | 4 +- 3 files changed, 164 insertions(+), 157 deletions(-) diff --git a/src/acc/acc_ml_optimiser_impl.h b/src/acc/acc_ml_optimiser_impl.h index b79aaaa54..cdcd41c27 100644 --- a/src/acc/acc_ml_optimiser_impl.h +++ b/src/acc/acc_ml_optimiser_impl.h @@ -2108,12 +2108,12 @@ template void storeWeightedSums(OptimisationParamters &op, SamplingParameters &sp, MlOptimiser *baseMLO, MlClass *accMLO, - std::vector &FinePassWeights, + IndexedDataArray &FinePassWeights, std::vector &ProjectionData, - std::vector > &FPCMasks, + std::vector &FPCMasks, AccPtrFactory ptrFactory, int ibody, - std::vector< AccPtrBundle > &bundleSWS) + AccPtrBundle &bundleSWS) { #ifdef TIMING if (op.part_id == baseMLO->exp_my_first_part_id) @@ -2240,19 +2240,21 @@ void storeWeightedSums(OptimisationParamters &op, SamplingParameters &sp, continue; // Use the constructed mask to construct a partial class-specific input - IndexedDataArray thisClassFinePassWeights(FinePassWeights[img_id],FPCMasks[img_id][exp_iclass]); + IndexedDataArray thisClassFinePassWeights(FinePassWeights,FPCMasks[exp_iclass]); // Re-define the job-partition of the indexedArray of weights so that the collect-kernel can work with it. - block_nums[nr_fake_classes*img_id + fake_class] = makeJobsForCollect(thisClassFinePassWeights, FPCMasks[img_id][exp_iclass], ProjectionData[img_id].orientation_num[exp_iclass]); + //TODO: should img_id be removed from the below? + XXXXX + block_nums[nr_fake_classes*img_id + fake_class] = makeJobsForCollect(thisClassFinePassWeights, FPCMasks[exp_iclass], ProjectionData[img_id].orientation_num[exp_iclass]); - bundleSWS[img_id].pack(FPCMasks[img_id][exp_iclass].jobOrigin); - bundleSWS[img_id].pack(FPCMasks[img_id][exp_iclass].jobExtent); + bundleSWS.pack(FPCMasks[exp_iclass].jobOrigin); + bundleSWS.pack(FPCMasks[exp_iclass].jobExtent); sumBlockNum+=block_nums[nr_fake_classes*img_id + fake_class]; RFLOAT myprior_x, myprior_y, myprior_z, old_offset_z; - RFLOAT old_offset_x = XX(op.old_offset[img_id]); - RFLOAT old_offset_y = YY(op.old_offset[img_id]); + RFLOAT old_offset_x = XX(op.old_offset); + RFLOAT old_offset_y = YY(op.old_offset); if (baseMLO->mymodel.ref_dim == 2 && baseMLO->mymodel.nr_bodies == 1) { @@ -2261,12 +2263,12 @@ void storeWeightedSums(OptimisationParamters &op, SamplingParameters &sp, } else { - myprior_x = XX(op.prior[img_id]); - myprior_y = YY(op.prior[img_id]); + myprior_x = XX(op.prior); + myprior_y = YY(op.prior); if (baseMLO->mymodel.data_dim == 3) { - myprior_z = ZZ(op.prior[img_id]); - old_offset_z = ZZ(op.old_offset[img_id]); + myprior_z = ZZ(op.prior); + old_offset_z = ZZ(op.old_offset); } } @@ -2314,7 +2316,7 @@ void storeWeightedSums(OptimisationParamters &op, SamplingParameters &sp, } } - bundleSWS[img_id].cpToDevice(); + bundleSWS.cpToDevice(); oo_otrans.cpToDevice(); DEBUG_HANDLE_ERROR(cudaStreamSynchronize(cudaStreamPerThread)); @@ -2341,7 +2343,7 @@ void storeWeightedSums(OptimisationParamters &op, SamplingParameters &sp, continue; // Use the constructed mask to construct a partial class-specific input - IndexedDataArray thisClassFinePassWeights(FinePassWeights[img_id],FPCMasks[img_id][exp_iclass]); + IndexedDataArray thisClassFinePassWeights(FinePassWeights,FPCMasks[exp_iclass]); long int cpos=fake_class*nr_transes; int block_num = block_nums[nr_fake_classes*img_id + fake_class]; @@ -2352,8 +2354,8 @@ void storeWeightedSums(OptimisationParamters &op, SamplingParameters &sp, &(~oo_otrans)[otrans_z+cpos], // otrans-size -> make const &(~oo_otrans)[otrans_x2y2z2+cpos], // otrans-size -> make const ~thisClassFinePassWeights.weights, - (XFLOAT)op.significant_weight[img_id], - (XFLOAT)op.sum_weight[img_id], + (XFLOAT)op.significant_weight, + (XFLOAT)op.sum_weight, sp.nr_trans, sp.nr_oversampled_trans, sp.nr_oversampled_rot, @@ -2366,8 +2368,8 @@ void storeWeightedSums(OptimisationParamters &op, SamplingParameters &sp, &(~p_thr_wsum_prior_offsetxyz_class)[sigma2_offset+partial_pos], ~thisClassFinePassWeights.rot_idx, ~thisClassFinePassWeights.trans_idx, - ~FPCMasks[img_id][exp_iclass].jobOrigin, - ~FPCMasks[img_id][exp_iclass].jobExtent, + ~FPCMasks[exp_iclass].jobOrigin, + ~FPCMasks[exp_iclass].jobExtent, accMLO->dataIs3D); LAUNCH_PRIVATE_ERROR(cudaGetLastError(),accMLO->errorStatus); @@ -2390,7 +2392,7 @@ void storeWeightedSums(OptimisationParamters &op, SamplingParameters &sp, for (long int n = partial_pos; n < partial_pos+block_num; n++) { - iorient= FinePassWeights[img_id].rot_id[FPCMasks[img_id][iclass].jobOrigin[n-partial_pos]+FPCMasks[img_id][iclass].firstPos]; + iorient= FinePassWeights.rot_id[FPCMasks[iclass].jobOrigin[n-partial_pos]+FPCMasks[iclass].firstPos]; long int mydir, idir=floor(iorient/sp.nr_psi); if (baseMLO->mymodel.orientational_prior_mode == NOPRIOR) @@ -2401,7 +2403,7 @@ void storeWeightedSums(OptimisationParamters &op, SamplingParameters &sp, // store partials according to indices of the relevant dimension unsigned ithr_wsum_pdf_direction = baseMLO->mymodel.nr_bodies > 1 ? ibody : iclass; DIRECT_MULTIDIM_ELEM(thr_wsum_pdf_direction[ithr_wsum_pdf_direction], mydir) += p_weights[n]; - thr_sumw_group[img_id] += p_weights[n]; + thr_sumw_group += p_weights[n]; thr_wsum_pdf_class[iclass] += p_weights[n]; thr_wsum_sigma2_offset += my_pixel_size * my_pixel_size * p_thr_wsum_prior_offsetxyz_class[sigma2_offset+n]; @@ -2422,78 +2424,75 @@ void storeWeightedSums(OptimisationParamters &op, SamplingParameters &sp, ======================================================*/ std::vector< RFLOAT> oversampled_rot, oversampled_tilt, oversampled_psi; - for (long int img_id = 0; img_id < sp.nr_images; img_id++) - { - RFLOAT my_pixel_size = baseMLO->mydata.getImagePixelSize(op.part_id, img_id); + RFLOAT my_pixel_size = baseMLO->mydata.getImagePixelSize(op.part_id, 0); - CTIC(accMLO->timer,"setMetadata"); + CTIC(accMLO->timer,"setMetadata"); - if(baseMLO->adaptive_oversampling!=0) - op.max_index[img_id].fineIndexToFineIndices(sp); // set partial indices corresponding to the found max_index, to be used below - else - op.max_index[img_id].coarseIndexToCoarseIndices(sp); + if(baseMLO->adaptive_oversampling!=0) + op.max_index.fineIndexToFineIndices(sp); // set partial indices corresponding to the found max_index, to be used below + else + op.max_index.coarseIndexToCoarseIndices(sp); - baseMLO->sampling.getTranslationsInPixel(op.max_index[img_id].itrans, baseMLO->adaptive_oversampling, my_pixel_size, - oversampled_translations_x, oversampled_translations_y, oversampled_translations_z, - (baseMLO->do_helical_refine) && (! baseMLO->ignore_helical_symmetry)); + baseMLO->sampling.getTranslationsInPixel(op.max_index.itrans, baseMLO->adaptive_oversampling, my_pixel_size, + oversampled_translations_x, oversampled_translations_y, oversampled_translations_z, + (baseMLO->do_helical_refine) && (! baseMLO->ignore_helical_symmetry)); - //TODO We already have rot, tilt and psi don't calculated them again - if(baseMLO->do_skip_align || baseMLO->do_skip_rotate) - baseMLO->sampling.getOrientations(sp.idir_min, sp.ipsi_min, baseMLO->adaptive_oversampling, oversampled_rot, oversampled_tilt, oversampled_psi, - op.pointer_dir_nonzeroprior, op.directions_prior, op.pointer_psi_nonzeroprior, op.psi_prior); - else - baseMLO->sampling.getOrientations(op.max_index[img_id].idir, op.max_index[img_id].ipsi, baseMLO->adaptive_oversampling, oversampled_rot, oversampled_tilt, oversampled_psi, - op.pointer_dir_nonzeroprior, op.directions_prior, op.pointer_psi_nonzeroprior, op.psi_prior); - - baseMLO->sampling.getOrientations(op.max_index[img_id].idir, op.max_index[img_id].ipsi, baseMLO->adaptive_oversampling, oversampled_rot, oversampled_tilt, oversampled_psi, - op.pointer_dir_nonzeroprior, op.directions_prior, op.pointer_psi_nonzeroprior, op.psi_prior); - - RFLOAT rot = oversampled_rot[op.max_index[img_id].ioverrot]; - RFLOAT tilt = oversampled_tilt[op.max_index[img_id].ioverrot]; - RFLOAT psi = oversampled_psi[op.max_index[img_id].ioverrot]; - - int icol_rot = (baseMLO->mymodel.nr_bodies == 1) ? METADATA_ROT : 0 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; - int icol_tilt = (baseMLO->mymodel.nr_bodies == 1) ? METADATA_TILT : 1 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; - int icol_psi = (baseMLO->mymodel.nr_bodies == 1) ? METADATA_PSI : 2 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; - int icol_xoff = (baseMLO->mymodel.nr_bodies == 1) ? METADATA_XOFF : 3 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; - int icol_yoff = (baseMLO->mymodel.nr_bodies == 1) ? METADATA_YOFF : 4 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; - int icol_zoff = (baseMLO->mymodel.nr_bodies == 1) ? METADATA_ZOFF : 5 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; - - RFLOAT old_rot = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, icol_rot); - DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, icol_rot) = rot; - RFLOAT old_tilt = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, icol_tilt); - DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, icol_tilt) = tilt; - RFLOAT old_psi = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, icol_psi); - DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, icol_psi) = psi; - - Matrix1D shifts(baseMLO->mymodel.data_dim); - - XX(shifts) = XX(op.old_offset[img_id]) + oversampled_translations_x[op.max_index[img_id].iovertrans]; - YY(shifts) = YY(op.old_offset[img_id]) + oversampled_translations_y[op.max_index[img_id].iovertrans]; - if (accMLO->dataIs3D) - { - ZZ(shifts) = ZZ(op.old_offset[img_id]) + oversampled_translations_z[op.max_index[img_id].iovertrans]; - } + //TODO We already have rot, tilt and psi don't calculated them again + if(baseMLO->do_skip_align || baseMLO->do_skip_rotate) + baseMLO->sampling.getOrientations(sp.idir_min, sp.ipsi_min, baseMLO->adaptive_oversampling, oversampled_rot, oversampled_tilt, oversampled_psi, + op.pointer_dir_nonzeroprior, op.directions_prior, op.pointer_psi_nonzeroprior, op.psi_prior); + else + baseMLO->sampling.getOrientations(op.max_index.idir, op.max_index.ipsi, baseMLO->adaptive_oversampling, oversampled_rot, oversampled_tilt, oversampled_psi, + op.pointer_dir_nonzeroprior, op.directions_prior, op.pointer_psi_nonzeroprior, op.psi_prior); + + baseMLO->sampling.getOrientations(op.max_index.idir, op.max_index.ipsi, baseMLO->adaptive_oversampling, oversampled_rot, oversampled_tilt, oversampled_psi, + op.pointer_dir_nonzeroprior, op.directions_prior, op.pointer_psi_nonzeroprior, op.psi_prior); + + RFLOAT rot = oversampled_rot[op.max_index.ioverrot]; + RFLOAT tilt = oversampled_tilt[op.max_index.ioverrot]; + RFLOAT psi = oversampled_psi[op.max_index.ioverrot]; + + int icol_rot = (baseMLO->mymodel.nr_bodies == 1) ? METADATA_ROT : 0 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; + int icol_tilt = (baseMLO->mymodel.nr_bodies == 1) ? METADATA_TILT : 1 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; + int icol_psi = (baseMLO->mymodel.nr_bodies == 1) ? METADATA_PSI : 2 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; + int icol_xoff = (baseMLO->mymodel.nr_bodies == 1) ? METADATA_XOFF : 3 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; + int icol_yoff = (baseMLO->mymodel.nr_bodies == 1) ? METADATA_YOFF : 4 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; + int icol_zoff = (baseMLO->mymodel.nr_bodies == 1) ? METADATA_ZOFF : 5 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; + + RFLOAT old_rot = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, icol_rot); + DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, icol_rot) = rot; + RFLOAT old_tilt = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, icol_tilt); + DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, icol_tilt) = tilt; + RFLOAT old_psi = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, icol_psi); + DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, icol_psi) = psi; + + Matrix1D shifts(baseMLO->mymodel.data_dim); + + XX(shifts) = XX(op.old_offset) + oversampled_translations_x[op.max_index.iovertrans]; + YY(shifts) = YY(op.old_offset) + oversampled_translations_y[op.max_index.iovertrans]; + if (accMLO->dataIs3D) + { + ZZ(shifts) = ZZ(op.old_offset) + oversampled_translations_z[op.max_index.iovertrans]; + } - // Use oldpsi-angle to rotate back the XX(exp_old_offset[img_id]) + oversampled_translations_x[iover_trans] and - if ( (baseMLO->do_helical_refine) && (! baseMLO->ignore_helical_symmetry) ) - transformCartesianAndHelicalCoords(shifts, shifts, old_rot, old_tilt, old_psi, HELICAL_TO_CART_COORDS); + // Use oldpsi-angle to rotate back the XX(exp_old_offset[img_id]) + oversampled_translations_x[iover_trans] and + if ( (baseMLO->do_helical_refine) && (! baseMLO->ignore_helical_symmetry) ) + transformCartesianAndHelicalCoords(shifts, shifts, old_rot, old_tilt, old_psi, HELICAL_TO_CART_COORDS); - DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, icol_xoff) = XX(shifts); - DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, icol_yoff) = YY(shifts); - if (accMLO->dataIs3D) - DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, icol_zoff) = ZZ(shifts); + DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, icol_xoff) = XX(shifts); + DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, icol_yoff) = YY(shifts); + if (accMLO->dataIs3D) + DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, icol_zoff) = ZZ(shifts); - if (ibody == 0) - { - DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_CLASS) = (RFLOAT)op.max_index[img_id].iclass + 1; - RFLOAT pmax = op.max_weight[img_id]/op.sum_weight[img_id]; - if(pmax>1) //maximum normalised probability weight is (unreasonably) larger than unity - CRITICAL("Relion is finding a normalised probability greater than 1"); - DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_PMAX) = pmax; - } - CTOC(accMLO->timer,"setMetadata"); - } + if (ibody == 0) + { + DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_CLASS) = (RFLOAT)op.max_index.iclass + 1; + RFLOAT pmax = op.max_weight/op.sum_weight; + if(pmax>1) //maximum normalised probability weight is (unreasonably) larger than unity + CRITICAL("Relion is finding a normalised probability greater than 1"); + DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_PMAX) = pmax; + } + CTOC(accMLO->timer,"setMetadata"); CTOC(accMLO->timer,"collect_data_2"); @@ -2507,7 +2506,6 @@ void storeWeightedSums(OptimisationParamters &op, SamplingParameters &sp, for (int img_id = 0; img_id < sp.nr_images; img_id++) { - int my_metadata_offset = op.metadata_offset + img_id; int group_id = baseMLO->mydata.getGroupId(op.part_id, img_id); const int optics_group = baseMLO->mydata.getOpticsGroup(op.part_id, img_id); RFLOAT my_pixel_size = baseMLO->mydata.getImagePixelSize(op.part_id, img_id); @@ -2546,9 +2544,9 @@ void storeWeightedSums(OptimisationParamters &op, SamplingParameters &sp, if ( (baseMLO->do_helical_refine) && (! baseMLO->ignore_helical_symmetry) ) { - RFLOAT rot_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_ROT); - RFLOAT tilt_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_TILT); - RFLOAT psi_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_PSI); + RFLOAT rot_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_ROT); + RFLOAT tilt_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_TILT); + RFLOAT psi_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_PSI); transformCartesianAndHelicalCoords(xshift, yshift, zshift, xshift, yshift, zshift, rot_deg, tilt_deg, psi_deg, (accMLO->dataIs3D) ? (3) : (2), HELICAL_TO_CART_COORDS); } @@ -2692,7 +2690,7 @@ void storeWeightedSums(OptimisationParamters &op, SamplingParameters &sp, continue; // Use the constructed mask to construct a partial class-specific input - IndexedDataArray thisClassFinePassWeights(FinePassWeights[img_id],FPCMasks[img_id][iclass]); + IndexedDataArray thisClassFinePassWeights(FinePassWeights,FPCMasks[iclass]); CTIC(accMLO->timer,"thisClassProjectionSetupCoarse"); // use "slice" constructor with class-specific parameters to retrieve a temporary ProjectionParams with data for this class @@ -2911,7 +2909,7 @@ void storeWeightedSums(OptimisationParamters &op, SamplingParameters &sp, ~Minvsigma2s, ~ctfs, translation_num, - (XFLOAT) op.significant_weight[img_id], + (XFLOAT) op.significant_weight, (XFLOAT) (baseMLO->is_som_iter ? class_sum_weight[iclass] : op.sum_weight[img_id]), ~eulers[iclass], op.local_Minvsigma2[img_id].xdim, @@ -2989,7 +2987,6 @@ void storeWeightedSums(OptimisationParamters &op, SamplingParameters &sp, for (int img_id = 0; img_id < sp.nr_images; img_id++) { - int my_metadata_offset = op.metadata_offset + img_id; int group_id = baseMLO->mydata.getGroupId(op.part_id, img_id); const int optics_group = baseMLO->mydata.getOpticsGroup(op.part_id, img_id); RFLOAT my_pixel_size = baseMLO->mydata.getOpticsPixelSize(optics_group); @@ -3007,7 +3004,10 @@ void storeWeightedSums(OptimisationParamters &op, SamplingParameters &sp, // Multiply by old value because the old norm_correction term was already applied to the image if (baseMLO->do_norm_correction && baseMLO->mymodel.nr_bodies == 1) { - RFLOAT old_norm_correction = DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_NORM); + xxxxxx + // TODO: think on how to combine all img_id into one norm correction!! + + RFLOAT old_norm_correction = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_NORM); old_norm_correction /= baseMLO->mymodel.avg_norm_correction; // The factor two below is because exp_wsum_norm_correctiom is similar to sigma2_noise, which is the variance for the real/imag components // The variance of the total image (on which one normalizes) is twice this value! @@ -3015,13 +3015,13 @@ void storeWeightedSums(OptimisationParamters &op, SamplingParameters &sp, thr_avg_norm_correction += normcorr; // Now set the new norm_correction in the relevant position of exp_metadata - DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_NORM) = normcorr; + DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_NORM) = normcorr; // Print warning for strange norm-correction values - if (!((baseMLO->iter == 1 && baseMLO->do_firstiter_cc) || baseMLO->do_always_cc) && DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_NORM) > 10.) + if (!((baseMLO->iter == 1 && baseMLO->do_firstiter_cc) || baseMLO->do_always_cc) && DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_NORM) > 10.) { - std::cout << " WARNING: norm_correction= "<< DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_NORM) + std::cout << " WARNING: norm_correction= "<< DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_NORM) << " for particle " << op.part_id << " in group " << group_id + 1 << "; Are your groups large enough? Or is the reference on the correct greyscale?" << std::endl; } @@ -3031,7 +3031,10 @@ void storeWeightedSums(OptimisationParamters &op, SamplingParameters &sp, // Store weighted sums for scale_correction if (baseMLO->do_scale_correction) { - // Divide XA by the old scale_correction and AA by the square of that, because was incorporated into Fctf + xxxxxx + // TODO: think on how to combine all img_id into one scale correction!! + + // Divide XA by the old scale_correction and AA by the square of that, because was incorporated into Fctf exp_wsum_scale_correction_XA[img_id] /= baseMLO->mymodel.scale_correction[group_id]; exp_wsum_scale_correction_AA[img_id] /= baseMLO->mymodel.scale_correction[group_id] * baseMLO->mymodel.scale_correction[group_id]; @@ -3053,20 +3056,29 @@ void storeWeightedSums(OptimisationParamters &op, SamplingParameters &sp, } RFLOAT dLL; + xxxx + // TODO: think about how to combine dLL sum_Pmax etc for all img_id + if ((baseMLO->iter==1 && baseMLO->do_firstiter_cc) || baseMLO->do_always_cc) - dLL = -op.min_diff2[img_id]; + dLL = -op.min_diff2; else - dLL = log(op.sum_weight[img_id]) - op.min_diff2[img_id] - logsigma2; + dLL = log(op.sum_weight) - op.min_diff2 - logsigma2; // Store dLL of each image in the output array, and keep track of total sum - DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_DLL) = dLL; + DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_DLL) = dLL; thr_sum_dLL += dLL; // Also store sum of Pmax - thr_sum_Pmax += DIRECT_A2D_ELEM(baseMLO->exp_metadata, my_metadata_offset, METADATA_PMAX); + thr_sum_Pmax += DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_PMAX); } + + + XXXXX Was here when leaving 6July2022: perhaps look first at legacy code? + + + // Now, inside a global_mutex, update the other weighted sums among all threads #pragma omp critical(AccMLO_global) { @@ -3224,7 +3236,8 @@ void accDoExpectationOneParticle(MlClass *myInstance, unsigned long part_id_sort for (long int iori = baseMLO->exp_my_first_part_id; iori <= baseMLO->exp_my_last_part_id; iori++) { if (iori == part_id_sorted) break; - op.metadata_offset += baseMLO->mydata.numberOfImagesInParticle(iori); + op.imagedata_offset += baseMLO->mydata.numberOfImagesInParticle(iori); + op.metadata_offset += 1; } #ifdef TIMING // Only time one thread @@ -3266,7 +3279,7 @@ baseMLO->timer.toc(baseMLO->TIMING_ESP_DIFF2_A); } // Initialise significant weight to minus one, so that all coarse sampling points will be handled in the first pass - op.significant_weight.resize(sp.nr_images, -1.); + op.significant_weight = -1.; // Only perform a second pass when using adaptive oversampling //int nr_sampling_passes = (baseMLO->adaptive_oversampling > 0) ? 2 : 1; @@ -3275,19 +3288,19 @@ baseMLO->timer.toc(baseMLO->TIMING_ESP_DIFF2_A); /// -- This is a iframe-indexed vector, each entry of which is a dense data-array. These are replacements to using // Mweight in the sparse (Fine-sampled) pass, coarse is unused but created empty input for convert ( FIXME ) - std::vector CoarsePassWeights(1, ptrFactory); - std::vector FinePassWeights(sp.nr_images, ptrFactory); + IndexedDataArray CoarsePassWeights(ptrFactory); + IndexedDataArray FinePassWeights(ptrFactory); // -- This is a iframe-indexed vector, each entry of which is a class-indexed vector of masks, one for each // class in FinePassWeights - std::vector < std::vector > FinePassClassMasks(sp.nr_images, std::vector (baseMLO->mymodel.nr_classes, ptrFactory)); + std::vector FinePassClassMasks( std::vector (baseMLO->mymodel.nr_classes, ptrFactory)); // -- This is a iframe-indexed vector, each entry of which is parameters used in the projection-operations *after* the // coarse pass, declared here to keep scope to storeWS std::vector < ProjectionParams > FineProjectionData(sp.nr_images, baseMLO->mymodel.nr_classes); - std::vector < AccPtrBundle > bundleD2(sp.nr_images, ptrFactory.makeBundle()); - std::vector < AccPtrBundle > bundleSWS(sp.nr_images, ptrFactory.makeBundle()); + AccPtrBundle bundleD2( ptrFactory.makeBundle()); + AccPtrBundle bundleSWS(ptrFactory.makeBundle()); for (int ipass = 0; ipass < nr_sampling_passes; ipass++) { @@ -3312,17 +3325,17 @@ if (thread_id == 0) baseMLO->timer.toc(baseMLO->TIMING_ESP_DIFF2_B); #endif - op.min_diff2.resize(sp.nr_images, 0); + op.min_diff2 = 0.; if (ipass == 0) { unsigned long weightsPerPart(baseMLO->mymodel.nr_classes * sp.nr_dir * sp.nr_psi * sp.nr_trans * sp.nr_oversampled_rot * sp.nr_oversampled_trans); - op.Mweight.resizeNoCp(1,1,sp.nr_images, weightsPerPart); + op.Mweight.resizeNoCp(1,1,1, weightsPerPart); AccPtr Mweight = ptrFactory.make(); - Mweight.setSize(sp.nr_images * weightsPerPart); + Mweight.setSize(weightsPerPart); Mweight.setHostPtr(op.Mweight.data); Mweight.deviceAlloc(); deviceInitValue(Mweight, -std::numeric_limits::max()); @@ -3346,33 +3359,33 @@ baseMLO->timer.tic(baseMLO->TIMING_ESP_DIFF2_D); // // -- go through all classes and generate projectionsetups for all classes - to be used in getASDF and storeWS below -- // // the reason to do this globally is subtle - we want the orientation_num of all classes to estimate a largest possible // // weight-array, which would be insanely much larger than necessary if we had to assume the worst. - for (int img_id = 0; img_id < sp.nr_images; img_id++) - { - FineProjectionData[img_id].orientationNumAllClasses = 0; - for (int exp_iclass = sp.iclass_min; exp_iclass <= sp.iclass_max; exp_iclass++) - { - if(exp_iclass>0) - FineProjectionData[img_id].class_idx[exp_iclass] = FineProjectionData[img_id].rots.size(); - FineProjectionData[img_id].class_entries[exp_iclass] = 0; - - CTIC(timer,"generateProjectionSetup"); - FineProjectionData[img_id].orientationNumAllClasses += generateProjectionSetupFine( - op, - sp, - baseMLO, - exp_iclass, - FineProjectionData[img_id]); - CTOC(timer,"generateProjectionSetup"); + for (int img_id = 0; img_id < sp.nr_images; img_id++) { + FineProjectionData[img_id].orientationNumAllClasses = 0; + for (int exp_iclass = sp.iclass_min; exp_iclass <= sp.iclass_max; exp_iclass++) { + if (exp_iclass > 0) + FineProjectionData[img_id].class_idx[exp_iclass] = FineProjectionData[img_id].rots.size(); + FineProjectionData[img_id].class_entries[exp_iclass] = 0; + + CTIC(timer, "generateProjectionSetup"); + FineProjectionData[img_id].orientationNumAllClasses += generateProjectionSetupFine( + op, + sp, + baseMLO, + exp_iclass, + FineProjectionData[img_id]); + CTOC(timer, "generateProjectionSetup"); - } - //set a maximum possible size for all weights (to be reduced by significance-checks) - size_t dataSize = FineProjectionData[img_id].orientationNumAllClasses*sp.nr_trans*sp.nr_oversampled_trans; - FinePassWeights[img_id].setDataSize(dataSize); - FinePassWeights[img_id].dual_alloc_all(); + } + } + //set a maximum possible size for all weights (to be reduced by significance-checks) + // SHWS 6Jul2022: assume dataSize of FineProjectionData is the same for all img_id + size_t dataSize = FineProjectionData[0].orientationNumAllClasses*sp.nr_trans*sp.nr_oversampled_trans; + FinePassWeights.setDataSize(dataSize); + FinePassWeights.dual_alloc_all(); + + bundleD2.setSize(2*(FineProjectionData[0].orientationNumAllClasses*sp.nr_trans*sp.nr_oversampled_trans)*sizeof(unsigned long)); + bundleD2.allAlloc(); - bundleD2[img_id].setSize(2*(FineProjectionData[img_id].orientationNumAllClasses*sp.nr_trans*sp.nr_oversampled_trans)*sizeof(unsigned long)); - bundleD2[img_id].allAlloc(); - } #ifdef TIMING // Only time one thread if (thread_id == 0) @@ -3404,11 +3417,9 @@ baseMLO->timer.tic(baseMLO->TIMING_ESP_DIFF2_E); // as of 3.1, no longer necessary? sp.current_image_size = baseMLO->mymodel.current_size; - for (unsigned long img_id = 0; img_id < sp.nr_images; img_id++) - { - bundleSWS[img_id].setSize(2*(FineProjectionData[img_id].orientationNumAllClasses)*sizeof(unsigned long)); - bundleSWS[img_id].allAlloc(); - } + // SHWS6Jul2022: assume FineProjectionData has same size for all img_id, just take first one here + bundleSWS.setSize(2*(FineProjectionData[0].orientationNumAllClasses)*sizeof(unsigned long)); + bundleSWS.allAlloc(); #ifdef TIMING // Only time one thread @@ -3419,10 +3430,7 @@ baseMLO->timer.toc(baseMLO->TIMING_ESP_DIFF2_E); storeWeightedSums(op, sp, baseMLO, myInstance, FinePassWeights, FineProjectionData, FinePassClassMasks, ptrFactory, ibody, bundleSWS); CTOC(timer,"storeWeightedSums"); - for (long int img_id = 0; img_id < sp.nr_images; img_id++) - { - FinePassWeights[img_id].dual_free_all(); - } + FinePassWeights.dual_free_all(); } CTOC(timer,"oneParticle"); diff --git a/src/ml_optimiser.cpp b/src/ml_optimiser.cpp index a2bc5a5a0..33e2bd34b 100644 --- a/src/ml_optimiser.cpp +++ b/src/ml_optimiser.cpp @@ -4059,6 +4059,7 @@ void MlOptimiser::expectationOneParticle(long int part_id_sorted, int thread_id) // In the first iteration, multiple seeds will be generated // A single random class is selected for each pool of images, and one does not marginalise over the orientations + // A single random class is selected for each pool of images, and one does not marginalise over the orientations // The optimal orientation is based on signal-product (rather than the signal-intensity sensitive Gaussian) // If do_firstiter_cc, then first perform a single iteration with K=1 and cross-correlation criteria, afterwards @@ -4119,12 +4120,12 @@ void MlOptimiser::expectationOneParticle(long int part_id_sorted, int thread_id) std::vector exp_pointer_dir_nonzeroprior, exp_pointer_psi_nonzeroprior; std::vector exp_directions_prior, exp_psi_prior, exp_local_sqrtXi2; int exp_current_image_size, exp_current_oversampling; - std::vector exp_highres_Xi2_img, exp_min_diff2; + std::vector exp_highres_Xi2_img; MultidimArray exp_Mweight; MultidimArray exp_Mcoarse_significant; // And from storeWeightedSums - std::vector exp_sum_weight, exp_significant_weight, exp_max_weight; - std::vector > exp_old_offset, exp_prior; + RFLOAT exp_min_diff2, exp_sum_weight, exp_significant_weight, exp_max_weight; + Matrix1D exp_old_offset, exp_prior; std::vector exp_wsum_norm_correction; std::vector > exp_power_imgs; @@ -4146,8 +4147,6 @@ void MlOptimiser::expectationOneParticle(long int part_id_sorted, int thread_id) exp_Fimg.resize(my_nr_images); exp_Fimg_nomask.resize(my_nr_images); exp_Fctf.resize(my_nr_images); - exp_old_offset.resize(my_nr_images); - exp_prior.resize(my_nr_images); if (mydata.is_3D) { @@ -4213,7 +4212,7 @@ void MlOptimiser::expectationOneParticle(long int part_id_sorted, int thread_id) } // Initialise significant weight to minus one, so that all coarse sampling points will be handled in the first pass - exp_significant_weight.resize(my_nr_images, -1.); + exp_significant_weight = -1.; // Only perform a second pass when using adaptive oversampling int nr_sampling_passes = (adaptive_oversampling > 0) ? 2 : 1; @@ -7853,14 +7852,14 @@ void MlOptimiser::storeWeightedSums(long int part_id, int ibody, int exp_current_oversampling, int metadata_offset, int exp_idir_min, int exp_idir_max, int exp_ipsi_min, int exp_ipsi_max, int exp_itrans_min, int exp_itrans_max, int exp_iclass_min, int exp_iclass_max, - std::vector &exp_min_diff2, + RFLOAT &exp_min_diff2, std::vector &exp_highres_Xi2_img, std::vector > &exp_Fimg, std::vector > &exp_Fimg_nomask, std::vector > &exp_Fctf, std::vector > &exp_power_img, - std::vector > &exp_old_offset, - std::vector > &exp_prior, + Matrix1D &exp_old_offset, + Matrix1D &exp_prior, MultidimArray &exp_Mweight, MultidimArray &exp_Mcoarse_significant, std::vector &exp_significant_weight, diff --git a/src/ml_optimiser.h b/src/ml_optimiser.h index 731273325..d4d8a636e 100644 --- a/src/ml_optimiser.h +++ b/src/ml_optimiser.h @@ -1118,7 +1118,7 @@ class MlOptimiser int exp_idir_min, int exp_idir_max, int exp_ipsi_min, int exp_ipsi_max, int exp_itrans_min, int exp_itrans_max, int my_iclass_min, int my_iclass_max, MultidimArray &exp_Mweight, MultidimArray &exp_Mcoarse_significant, - std::vector &exp_significant_weight, std::vector &exp_sum_weight, + RFLOAT &exp_significant_weight, RFLOAT &exp_sum_weight, Matrix1D &exp_old_offset, Matrix1D &exp_prior, RFLOAT &exp_min_diff2, std::vector &exp_pointer_dir_nonzeroprior, std::vector &exp_pointer_psi_nonzeroprior, std::vector &exp_directions_prior, std::vector &exp_psi_prior); @@ -1128,7 +1128,7 @@ class MlOptimiser int exp_current_oversampling, int metadata_offset, int exp_idir_min, int exp_idir_max, int exp_ipsi_min, int exp_ipsi_max, int exp_itrans_min, int exp_itrans_max, int my_iclass_min, int my_iclass_max, - std::vector &exp_min_diff2, + RFLOAT &exp_min_diff2, std::vector &exp_highres_Xi2_img, std::vector > &exp_Fimg, std::vector > &exp_Fimg_nomask, From 6afb57725f37f6a63e1112867b4c34fcdde59084 Mon Sep 17 00:00:00 2001 From: scheres Date: Thu, 7 Jul 2022 11:40:41 +0100 Subject: [PATCH 075/495] now storeWeightedSums done too, and code compiles... --- src/acc/acc_helper_functions_impl.h | 24 +- src/acc/acc_ml_optimiser.h | 7 +- src/acc/acc_ml_optimiser_impl.h | 355 ++++++++++------------ src/acc/cuda/cuda_kernels/diff2.cuh | 3 - src/ml_optimiser.cpp | 446 +++++++++++++--------------- src/ml_optimiser.h | 10 +- 6 files changed, 373 insertions(+), 472 deletions(-) diff --git a/src/acc/acc_helper_functions_impl.h b/src/acc/acc_helper_functions_impl.h index 33468138b..5aff46ec8 100644 --- a/src/acc/acc_helper_functions_impl.h +++ b/src/acc/acc_helper_functions_impl.h @@ -342,8 +342,8 @@ void runWavgKernel( wdiff2s_AA, wdiff2s_XA, translation_num, - (XFLOAT) op.sum_weight[img_id], - (XFLOAT) op.significant_weight[img_id], + (XFLOAT) op.sum_weight, + (XFLOAT) op.significant_weight, part_scale, stream ); @@ -364,8 +364,8 @@ void runWavgKernel( wdiff2s_AA, wdiff2s_XA, translation_num, - (XFLOAT) op.sum_weight[img_id], - (XFLOAT) op.significant_weight[img_id], + (XFLOAT) op.sum_weight, + (XFLOAT) op.significant_weight, part_scale, stream ); @@ -386,8 +386,8 @@ void runWavgKernel( wdiff2s_AA, wdiff2s_XA, translation_num, - (XFLOAT) op.sum_weight[img_id], - (XFLOAT) op.significant_weight[img_id], + (XFLOAT) op.sum_weight, + (XFLOAT) op.significant_weight, part_scale, stream ); @@ -411,8 +411,8 @@ void runWavgKernel( wdiff2s_AA, wdiff2s_XA, translation_num, - (XFLOAT) op.sum_weight[img_id], - (XFLOAT) op.significant_weight[img_id], + (XFLOAT) op.sum_weight, + (XFLOAT) op.significant_weight, part_scale, stream ); @@ -433,8 +433,8 @@ void runWavgKernel( wdiff2s_AA, wdiff2s_XA, translation_num, - (XFLOAT) op.sum_weight[img_id], - (XFLOAT) op.significant_weight[img_id], + (XFLOAT) op.sum_weight, + (XFLOAT) op.significant_weight, part_scale, stream ); @@ -455,8 +455,8 @@ void runWavgKernel( wdiff2s_AA, wdiff2s_XA, translation_num, - (XFLOAT) op.sum_weight[img_id], - (XFLOAT) op.significant_weight[img_id], + (XFLOAT) op.sum_weight, + (XFLOAT) op.significant_weight, part_scale, stream ); diff --git a/src/acc/acc_ml_optimiser.h b/src/acc/acc_ml_optimiser.h index b52fcf4e2..8777191be 100644 --- a/src/acc/acc_ml_optimiser.h +++ b/src/acc/acc_ml_optimiser.h @@ -137,7 +137,8 @@ class OptimisationParamters unsigned long part_id; std::vector > Fimg, Fimg_nomask, local_Fimgs_shifted, local_Fimgs_shifted_nomask; - std::vector > Fctf, local_Fctf, local_Minvsigma2, FstMulti; + std::vector > Fctf, local_Fctf, local_Minvsigma2; + MultidimArray FstMulti; std::vector pointer_dir_nonzeroprior, pointer_psi_nonzeroprior; std::vector directions_prior, psi_prior, local_sqrtXi2; std::vector highres_Xi2_img; @@ -145,7 +146,7 @@ class OptimisationParamters MultidimArray Mcoarse_significant; // And from storeWeightedSums RFLOAT sum_weight, significant_weight, max_weight; - std::vector< std::vector > sum_weight_class; + std::vector sum_weight_class; Matrix1D old_offset, prior; std::vector > power_img; MultidimArray Mweight; @@ -161,8 +162,6 @@ class OptimisationParamters Fimg.resize(nr_images); Fimg_nomask.resize(nr_images); Fctf.resize(nr_images); - max_index.resize(nr_images); - sum_weight_class.resize(nr_images); }; }; diff --git a/src/acc/acc_ml_optimiser_impl.h b/src/acc/acc_ml_optimiser_impl.h index cdcd41c27..dfd408eca 100644 --- a/src/acc/acc_ml_optimiser_impl.h +++ b/src/acc/acc_ml_optimiser_impl.h @@ -767,7 +767,7 @@ void getFourierTransformsAndCtfs(long int part_id, if ( NZYXSIZE(FstMulti) > 0 ) { baseMLO->applySubtomoCorrection(op.Fimg.at(img_id), op.Fimg_nomask.at(img_id), op.Fctf.at(img_id), FstMulti); - op.FstMulti.at(img_id) = FstMulti; + op.FstMulti = FstMulti; } // If we're doing multibody refinement, now subtract projections of the other bodies from both the masked and the unmasked particle @@ -929,7 +929,7 @@ void getAllSquaredDifferencesCoarse( std::vector > dummy; std::vector > > dummy2; - std::vector > dummyRF; + MultidimArray dummyRF; baseMLO->precalculateShiftedImagesCtfsAndInvSigma2s(false, false, op.part_id, sp.current_oversampling, op.metadata_offset, // inserted SHWS 12112015 sp.itrans_min, sp.itrans_max, op.Fimg, dummy, op.Fctf, dummy2, dummy2, op.local_Fctf, op.local_sqrtXi2, op.local_Minvsigma2, op.FstMulti, dummyRF); @@ -1118,7 +1118,7 @@ void getAllSquaredDifferencesCoarse( buildCorrImage(baseMLO,op,corr_img,img_id,group_id); corr_img.cpToDevice(); - // REPORT_ERROR("TODO: Dari needs to write a function to add op.highres_Xi2_img[img_id] / 2. to all Weights, not initialise again!"); + if (sp.nr_images > 1) REPORT_ERROR("TODO: Dari needs to write a function to add op.highres_Xi2_img[img_id] / 2. to all Weights, not initialise again!"); deviceInitValue(allWeights, (XFLOAT) (op.highres_Xi2_img[img_id] / 2.)); allWeights_pos = 0; @@ -1159,8 +1159,7 @@ void getAllSquaredDifferencesCoarse( do_CC, accMLO->dataIs3D); - if (accMLO->mydata.is_tomo) REPORT_ERROR("ERROR: TODO: think about img_id* multiplication below... I added += instead of = in the GPU and CPU kernels below!!!"); - mapAllWeightsToMweights( + mapAllWeightsToMweights( ~projectorPlans[iclass].iorientclasses, &(~allWeights)[allWeights_pos], &(~Mweight)[weightsPerPart], @@ -1219,7 +1218,7 @@ void getAllSquaredDifferencesFine( CTIC(accMLO->timer,"precalculateShiftedImagesCtfsAndInvSigma2s"); std::vector > dummy; std::vector > > dummy2; - std::vector > dummyRF; + MultidimArray dummyRF; baseMLO->precalculateShiftedImagesCtfsAndInvSigma2s(false, false, op.part_id, sp.current_oversampling, op.metadata_offset, // inserted SHWS 12112015 sp.itrans_min, sp.itrans_max, op.Fimg, dummy, op.Fctf, dummy2, dummy2, op.local_Fctf, op.local_sqrtXi2, op.local_Minvsigma2, op.FstMulti, dummyRF); @@ -1850,7 +1849,7 @@ void convertAllSquaredDifferencesToWeights(unsigned exp_ipass, { std::cerr << std::endl; std::cerr << " fn_img= " << sp.current_img << std::endl; - std::cerr " adaptive_fraction= " << baseMLO->adaptive_fraction << std::endl; + std::cerr << " adaptive_fraction= " << baseMLO->adaptive_fraction << std::endl; std::cerr << " min_diff2= " << op.min_diff2 << std::endl; pdf_orientation.dumpAccToFile("error_dump_pdf_orientation"); @@ -2124,10 +2123,8 @@ void storeWeightedSums(OptimisationParamters &op, SamplingParameters &sp, // Re-do below because now also want unmasked images AND if (stricht_highres_exp >0.) then may need to resize std::vector > dummy; std::vector > > dummy2; - std::vector > exp_local_STMulti; - bool do_subtomo_correction = op.FstMulti.size() > 0 && NZYXSIZE(op.FstMulti[0]) > 0; - if (do_subtomo_correction) - exp_local_STMulti.resize(sp.nr_images); + MultidimArray exp_local_STMulti; + bool do_subtomo_correction = NZYXSIZE(op.FstMulti) > 0; baseMLO->precalculateShiftedImagesCtfsAndInvSigma2s(false, true, op.part_id, sp.current_oversampling, op.metadata_offset, // inserted SHWS 12112015 sp.itrans_min, sp.itrans_max, op.Fimg, op.Fimg_nomask, op.Fctf, dummy2, dummy2, @@ -2142,41 +2139,16 @@ void storeWeightedSums(OptimisationParamters &op, SamplingParameters &sp, } // For norm_correction and scale_correction of all images of this particle - std::vector exp_wsum_norm_correction; - std::vector exp_wsum_scale_correction_XA, exp_wsum_scale_correction_AA; - std::vector thr_wsum_signal_product_spectra, thr_wsum_reference_power_spectra; - exp_wsum_norm_correction.resize(sp.nr_images, 0.); - std::vector > thr_wsum_sigma2_noise, thr_wsum_ctf2, thr_wsum_stMulti; - - // for noise estimation (per image) - thr_wsum_sigma2_noise.resize(sp.nr_images); - thr_wsum_ctf2.resize(sp.nr_images); - thr_wsum_stMulti.resize(sp.nr_images); - - // For scale_correction - if (baseMLO->do_scale_correction) - { - exp_wsum_scale_correction_XA.resize(sp.nr_images); - exp_wsum_scale_correction_AA.resize(sp.nr_images); - thr_wsum_signal_product_spectra.resize(sp.nr_images); - thr_wsum_reference_power_spectra.resize(sp.nr_images); - } + RFLOAT exp_wsum_norm_correction = 0.; + RFLOAT exp_wsum_scale_correction_XA = 0., exp_wsum_scale_correction_AA = 0.; + RFLOAT thr_wsum_signal_product_spectra = 0., thr_wsum_reference_power_spectra = 0.; + MultidimArray thr_wsum_sigma2_noise, thr_wsum_ctf2, thr_wsum_stMulti; - // Possibly different array sizes in different optics groups! - for (int img_id = 0; img_id < sp.nr_images; img_id++) - { - int optics_group = baseMLO->mydata.getOpticsGroup(op.part_id, img_id); - thr_wsum_sigma2_noise[img_id].initZeros(baseMLO->image_full_size[optics_group]/2 + 1); - thr_wsum_stMulti[img_id].initZeros(baseMLO->image_full_size[optics_group]/2 + 1); - thr_wsum_ctf2[img_id].initZeros(baseMLO->image_full_size[optics_group]/2 + 1); - if (baseMLO->do_scale_correction) - { - exp_wsum_scale_correction_AA[img_id] = 0.; - exp_wsum_scale_correction_XA[img_id] = 0.; - thr_wsum_signal_product_spectra[img_id] = 0.; - thr_wsum_reference_power_spectra[img_id] = 0.; - } - } + int optics_group = baseMLO->mydata.getOpticsGroup(op.part_id, 0); + thr_wsum_sigma2_noise.initZeros(baseMLO->image_full_size[optics_group]/2 + 1); + thr_wsum_ctf2.initZeros(baseMLO->image_full_size[optics_group]/2 + 1); + if (do_subtomo_correction) + thr_wsum_stMulti.initZeros(baseMLO->image_full_size[optics_group]/2 + 1); std::vector oversampled_translations_x, oversampled_translations_y, oversampled_translations_z; bool have_warned_small_scale = false; @@ -2184,14 +2156,14 @@ void storeWeightedSums(OptimisationParamters &op, SamplingParameters &sp, // Make local copies of weighted sums (except BPrefs, which are too big) // so that there are not too many mutex locks below std::vector > thr_wsum_pdf_direction; - std::vector thr_wsum_norm_correction, thr_sumw_group, thr_wsum_pdf_class, thr_wsum_prior_offsetx_class, thr_wsum_prior_offsety_class; - RFLOAT thr_wsum_sigma2_offset; + std::vector thr_wsum_norm_correction, thr_wsum_pdf_class, thr_wsum_prior_offsetx_class, thr_wsum_prior_offsety_class; + RFLOAT thr_wsum_sigma2_offset, thr_sumw_group; MultidimArray thr_metadata, zeroArray; // wsum_pdf_direction is a 1D-array (of length sampling.NrDirections()) for each class zeroArray.initZeros(baseMLO->sampling.NrDirections()); thr_wsum_pdf_direction.resize(baseMLO->mymodel.nr_classes * baseMLO->mymodel.nr_bodies, zeroArray); // sumw_group is a RFLOAT for each group - thr_sumw_group.resize(sp.nr_images, 0.); + thr_sumw_group= 0.; // wsum_pdf_class is a RFLOAT for each class thr_wsum_pdf_class.resize(baseMLO->mymodel.nr_classes, 0.); if (baseMLO->mymodel.ref_dim == 2) @@ -2243,8 +2215,6 @@ void storeWeightedSums(OptimisationParamters &op, SamplingParameters &sp, IndexedDataArray thisClassFinePassWeights(FinePassWeights,FPCMasks[exp_iclass]); // Re-define the job-partition of the indexedArray of weights so that the collect-kernel can work with it. - //TODO: should img_id be removed from the below? - XXXXX block_nums[nr_fake_classes*img_id + fake_class] = makeJobsForCollect(thisClassFinePassWeights, FPCMasks[exp_iclass], ProjectionData[img_id].orientation_num[exp_iclass]); bundleSWS.pack(FPCMasks[exp_iclass].jobOrigin); @@ -2831,17 +2801,17 @@ void storeWeightedSums(OptimisationParamters &op, SamplingParameters &sp, ======================================================*/ int nr_classes = baseMLO->mymodel.nr_classes; - std::vector class_sum_weight(nr_classes, baseMLO->is_som_iter ? 0 : op.sum_weight[img_id]); + std::vector class_sum_weight(nr_classes, baseMLO->is_som_iter ? 0 : op.sum_weight); if (baseMLO->is_som_iter) { - std::vector s = SomGraph::arg_sort(op.sum_weight_class[img_id], false); + std::vector s = SomGraph::arg_sort(op.sum_weight_class, false); unsigned bpu = s[0]; unsigned sbpu = s[1]; baseMLO->wsum_model.som.add_edge_activity(bpu, sbpu); - class_sum_weight[bpu] = op.sum_weight_class[img_id][bpu]; + class_sum_weight[bpu] = op.sum_weight_class[bpu]; thr_wsum_pdf_class[bpu] += 1; baseMLO->wsum_model.som.add_node_activity(bpu); baseMLO->mymodel.som.add_node_age(bpu); @@ -2851,7 +2821,7 @@ void storeWeightedSums(OptimisationParamters &op, SamplingParameters &sp, for (int i = 0; i < weights.size(); i++) { unsigned idx = weights[i].first; float w = weights[i].second * baseMLO->som_neighbour_pull; - class_sum_weight[idx] = op.sum_weight_class[img_id][idx] / w; + class_sum_weight[idx] = op.sum_weight_class[idx] / w; thr_wsum_pdf_class[idx] += w; baseMLO->wsum_model.som.add_node_activity(idx, w); baseMLO->mymodel.som.add_node_age(idx, w); @@ -2910,7 +2880,7 @@ void storeWeightedSums(OptimisationParamters &op, SamplingParameters &sp, ~ctfs, translation_num, (XFLOAT) op.significant_weight, - (XFLOAT) (baseMLO->is_som_iter ? class_sum_weight[iclass] : op.sum_weight[img_id]), + (XFLOAT) (baseMLO->is_som_iter ? class_sum_weight[iclass] : op.sum_weight), ~eulers[iclass], op.local_Minvsigma2[img_id].xdim, op.local_Minvsigma2[img_id].ydim, @@ -2957,8 +2927,8 @@ void storeWeightedSums(OptimisationParamters &op, SamplingParameters &sp, if (ires > -1 && baseMLO->do_scale_correction && DIRECT_A1D_ELEM(baseMLO->mymodel.data_vs_prior_class[exp_iclass], ires) > 3.) { - exp_wsum_scale_correction_AA[img_id] += wdiff2s[AA_offset+AAXA_pos+j]; - exp_wsum_scale_correction_XA[img_id] += wdiff2s[XA_offset+AAXA_pos+j]; + exp_wsum_scale_correction_AA += wdiff2s[AA_offset+AAXA_pos+j]; + exp_wsum_scale_correction_XA += wdiff2s[XA_offset+AAXA_pos+j]; } } AAXA_pos += image_size; @@ -2969,8 +2939,8 @@ void storeWeightedSums(OptimisationParamters &op, SamplingParameters &sp, int ires = DIRECT_MULTIDIM_ELEM(baseMLO->Mresol_fine[optics_group], j); if (ires > -1) { - thr_wsum_sigma2_noise[img_id].data[ires] += (RFLOAT) wdiff2s[sum_offset+j]; - exp_wsum_norm_correction[img_id] += (RFLOAT) wdiff2s[sum_offset+j]; //TODO could be gpu-reduced + thr_wsum_sigma2_noise.data[ires] += (RFLOAT) wdiff2s[sum_offset+j]; + exp_wsum_norm_correction += (RFLOAT) wdiff2s[sum_offset+j]; //TODO could be gpu-reduced } } @@ -2984,177 +2954,156 @@ void storeWeightedSums(OptimisationParamters &op, SamplingParameters &sp, // loop over all images inside this particle RFLOAT thr_avg_norm_correction = 0.; RFLOAT thr_sum_dLL = 0., thr_sum_Pmax = 0.; - for (int img_id = 0; img_id < sp.nr_images; img_id++) - { - - int group_id = baseMLO->mydata.getGroupId(op.part_id, img_id); - const int optics_group = baseMLO->mydata.getOpticsGroup(op.part_id, img_id); - RFLOAT my_pixel_size = baseMLO->mydata.getOpticsPixelSize(optics_group); - int my_image_size = baseMLO->mydata.getOpticsImageSize(optics_group); - - // If the current images were smaller than the original size, fill the rest of wsum_model.sigma2_noise with the power_class spectrum of the images - for (unsigned long ires = baseMLO->image_current_size[optics_group]/2 + 1; ires < baseMLO->image_full_size[optics_group]/2 + 1; ires++) - { - DIRECT_A1D_ELEM(thr_wsum_sigma2_noise[img_id], ires) += DIRECT_A1D_ELEM(op.power_img[img_id], ires); - // Also extend the weighted sum of the norm_correction - exp_wsum_norm_correction[img_id] += DIRECT_A1D_ELEM(op.power_img[img_id], ires); - } - - // Store norm_correction - // Multiply by old value because the old norm_correction term was already applied to the image - if (baseMLO->do_norm_correction && baseMLO->mymodel.nr_bodies == 1) - { - xxxxxx - // TODO: think on how to combine all img_id into one norm correction!! - - RFLOAT old_norm_correction = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_NORM); - old_norm_correction /= baseMLO->mymodel.avg_norm_correction; - // The factor two below is because exp_wsum_norm_correctiom is similar to sigma2_noise, which is the variance for the real/imag components - // The variance of the total image (on which one normalizes) is twice this value! - RFLOAT normcorr = old_norm_correction * sqrt(exp_wsum_norm_correction[img_id] * 2.); - thr_avg_norm_correction += normcorr; + int group_id = baseMLO->mydata.getGroupId(op.part_id, 0); + const int optics_group = baseMLO->mydata.getOpticsGroup(op.part_id, 0); + RFLOAT my_pixel_size = baseMLO->mydata.getOpticsPixelSize(optics_group); + int my_image_size = baseMLO->mydata.getOpticsImageSize(optics_group); - // Now set the new norm_correction in the relevant position of exp_metadata - DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_NORM) = normcorr; - - - // Print warning for strange norm-correction values - if (!((baseMLO->iter == 1 && baseMLO->do_firstiter_cc) || baseMLO->do_always_cc) && DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_NORM) > 10.) - { - std::cout << " WARNING: norm_correction= "<< DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_NORM) - << " for particle " << op.part_id << " in group " << group_id + 1 - << "; Are your groups large enough? Or is the reference on the correct greyscale?" << std::endl; - } - - } - - // Store weighted sums for scale_correction - if (baseMLO->do_scale_correction) - { - xxxxxx - // TODO: think on how to combine all img_id into one scale correction!! + // If the current images were smaller than the original size, fill the rest of wsum_model.sigma2_noise with the power_class spectrum of the images + for (int img_id = 0; img_id < sp.nr_images; img_id++) + { + for (unsigned long ires = baseMLO->image_current_size[optics_group] / 2 + 1; + ires < baseMLO->image_full_size[optics_group] / 2 + 1; ires++) { + DIRECT_A1D_ELEM(thr_wsum_sigma2_noise, ires) += DIRECT_A1D_ELEM(op.power_img[img_id], ires); + // Also extend the weighted sum of the norm_correction + exp_wsum_norm_correction += DIRECT_A1D_ELEM(op.power_img[img_id], ires); + } + } - // Divide XA by the old scale_correction and AA by the square of that, because was incorporated into Fctf - exp_wsum_scale_correction_XA[img_id] /= baseMLO->mymodel.scale_correction[group_id]; - exp_wsum_scale_correction_AA[img_id] /= baseMLO->mymodel.scale_correction[group_id] * baseMLO->mymodel.scale_correction[group_id]; + // Store norm_correction + // Multiply by old value because the old norm_correction term was already applied to the image + if (baseMLO->do_norm_correction && baseMLO->mymodel.nr_bodies == 1) + { - thr_wsum_signal_product_spectra[img_id] += exp_wsum_scale_correction_XA[img_id]; - thr_wsum_reference_power_spectra[img_id] += exp_wsum_scale_correction_AA[img_id]; - } + RFLOAT old_norm_correction = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_NORM); + old_norm_correction /= baseMLO->mymodel.avg_norm_correction; + // The factor two below is because exp_wsum_norm_correctiom is similar to sigma2_noise, which is the variance for the real/imag components + // The variance of the total image (on which one normalizes) is twice this value! + RFLOAT normcorr = old_norm_correction * sqrt(exp_wsum_norm_correction * 2.); + thr_avg_norm_correction += normcorr; - // Calculate DLL for each particle - RFLOAT logsigma2 = 0.; - RFLOAT remap_image_sizes = (baseMLO->mymodel.ori_size * baseMLO->mymodel.pixel_size) / (my_image_size * my_pixel_size); - FOR_ALL_DIRECT_ELEMENTS_IN_MULTIDIMARRAY(baseMLO->Mresol_fine[optics_group]) - { - int ires = DIRECT_MULTIDIM_ELEM(baseMLO->Mresol_fine[optics_group], n); - int ires_remapped = ROUND(remap_image_sizes * ires); - // Note there is no sqrt in the normalisation term because of the 2-dimensionality of the complex-plane - // Also exclude origin from logsigma2, as this will not be considered in the P-calculations - if (ires > 0 && ires_remapped < XSIZE(baseMLO->mymodel.sigma2_noise[optics_group])) - logsigma2 += log( 2. * PI * DIRECT_A1D_ELEM(baseMLO->mymodel.sigma2_noise[optics_group], ires_remapped)); - } - RFLOAT dLL; + // Now set the new norm_correction in the relevant position of exp_metadata + DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_NORM) = normcorr; - xxxx - // TODO: think about how to combine dLL sum_Pmax etc for all img_id + // Print warning for strange norm-correction values + if (!((baseMLO->iter == 1 && baseMLO->do_firstiter_cc) || baseMLO->do_always_cc) && DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_NORM) > 10.) + { + std::cout << " WARNING: norm_correction= "<< DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_NORM) + << " for particle " << op.part_id + << "; Are your groups large enough? Or is the reference on the correct greyscale?" << std::endl; + } - if ((baseMLO->iter==1 && baseMLO->do_firstiter_cc) || baseMLO->do_always_cc) - dLL = -op.min_diff2; - else - dLL = log(op.sum_weight) - op.min_diff2 - logsigma2; + } - // Store dLL of each image in the output array, and keep track of total sum - DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_DLL) = dLL; - thr_sum_dLL += dLL; + // Store weighted sums for scale_correction + if (baseMLO->do_scale_correction) + { - // Also store sum of Pmax - thr_sum_Pmax += DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_PMAX); + // Divide XA by the old scale_correction and AA by the square of that, because was incorporated into Fctf + exp_wsum_scale_correction_XA /= baseMLO->mymodel.scale_correction[group_id]; + exp_wsum_scale_correction_AA /= baseMLO->mymodel.scale_correction[group_id] * baseMLO->mymodel.scale_correction[group_id]; - } + thr_wsum_signal_product_spectra += exp_wsum_scale_correction_XA; + thr_wsum_reference_power_spectra += exp_wsum_scale_correction_AA; + } + // Calculate DLL for each particle + RFLOAT logsigma2 = 0.; + RFLOAT remap_image_sizes = (baseMLO->mymodel.ori_size * baseMLO->mymodel.pixel_size) / (my_image_size * my_pixel_size); + FOR_ALL_DIRECT_ELEMENTS_IN_MULTIDIMARRAY(baseMLO->Mresol_fine[optics_group]) + { + int ires = DIRECT_MULTIDIM_ELEM(baseMLO->Mresol_fine[optics_group], n); + int ires_remapped = ROUND(remap_image_sizes * ires); + // Note there is no sqrt in the normalisation term because of the 2-dimensionality of the complex-plane + // Also exclude origin from logsigma2, as this will not be considered in the P-calculations + if (ires > 0 && ires_remapped < XSIZE(baseMLO->mymodel.sigma2_noise[optics_group])) + logsigma2 += log( 2. * PI * DIRECT_A1D_ELEM(baseMLO->mymodel.sigma2_noise[optics_group], ires_remapped)); + } + RFLOAT dLL; + if ((baseMLO->iter==1 && baseMLO->do_firstiter_cc) || baseMLO->do_always_cc) + dLL = -op.min_diff2; + else + dLL = log(op.sum_weight) - op.min_diff2 - logsigma2; - XXXXX Was here when leaving 6July2022: perhaps look first at legacy code? + // Store dLL of each image in the output array, and keep track of total sum + DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_DLL) = dLL; + thr_sum_dLL += dLL; + // Also store sum of Pmax + thr_sum_Pmax += DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_PMAX); // Now, inside a global_mutex, update the other weighted sums among all threads #pragma omp critical(AccMLO_global) - { - for (int img_id = 0; img_id < sp.nr_images; img_id++) - { - long int igroup = baseMLO->mydata.getGroupId(op.part_id, img_id); - int optics_group = baseMLO->mydata.getOpticsGroup(op.part_id, img_id); + { + long int igroup = baseMLO->mydata.getGroupId(op.part_id, 0); + int optics_group = baseMLO->mydata.getOpticsGroup(op.part_id, 0); - if (baseMLO->mydata.obsModel.getCtfPremultiplied(optics_group)) - { - RFLOAT myscale = XMIPP_MAX(0.001, baseMLO->mymodel.scale_correction[igroup]); - FOR_ALL_DIRECT_ELEMENTS_IN_MULTIDIMARRAY(baseMLO->Mresol_fine[optics_group]) - { - int ires = DIRECT_MULTIDIM_ELEM(baseMLO->Mresol_fine[optics_group], n); - if (ires > -1) - DIRECT_MULTIDIM_ELEM(thr_wsum_ctf2[img_id], ires) += myscale * DIRECT_MULTIDIM_ELEM(op.local_Fctf[img_id], n); - } + if (baseMLO->mydata.obsModel.getCtfPremultiplied(optics_group)) { + RFLOAT myscale = XMIPP_MAX(0.001, baseMLO->mymodel.scale_correction[igroup]); + FOR_ALL_DIRECT_ELEMENTS_IN_MULTIDIMARRAY(baseMLO->Mresol_fine[optics_group]) { + int ires = DIRECT_MULTIDIM_ELEM(baseMLO->Mresol_fine[optics_group], n); + if (ires > -1) + DIRECT_MULTIDIM_ELEM(thr_wsum_ctf2, ires) += + myscale * DIRECT_MULTIDIM_ELEM(op.local_Fctf[0], n); } + } - if (do_subtomo_correction) - { - FOR_ALL_DIRECT_ELEMENTS_IN_MULTIDIMARRAY(baseMLO->Mresol_fine[optics_group]) - { - int ires = DIRECT_MULTIDIM_ELEM(baseMLO->Mresol_fine[optics_group], n); - if (ires > -1) - DIRECT_MULTIDIM_ELEM(thr_wsum_stMulti[img_id], ires) += DIRECT_MULTIDIM_ELEM(exp_local_STMulti[img_id], n); - } + if (do_subtomo_correction) { + FOR_ALL_DIRECT_ELEMENTS_IN_MULTIDIMARRAY(baseMLO->Mresol_fine[optics_group]) { + int ires = DIRECT_MULTIDIM_ELEM(baseMLO->Mresol_fine[optics_group], n); + if (ires > -1) + DIRECT_MULTIDIM_ELEM(thr_wsum_stMulti, ires) += DIRECT_MULTIDIM_ELEM( + exp_local_STMulti, n); } + } - int my_image_size = baseMLO->mydata.getOpticsImageSize(optics_group); - RFLOAT my_pixel_size = baseMLO->mydata.getOpticsPixelSize(optics_group); - RFLOAT remap_image_sizes = (baseMLO->mymodel.ori_size * baseMLO->mymodel.pixel_size) / (my_image_size * my_pixel_size); - FOR_ALL_DIRECT_ELEMENTS_IN_ARRAY1D(thr_wsum_sigma2_noise[img_id]) - { - int i_resam = ROUND(i * remap_image_sizes); - if (i_resam < XSIZE(baseMLO->wsum_model.sigma2_noise[optics_group])) - { - DIRECT_A1D_ELEM(baseMLO->wsum_model.sigma2_noise[optics_group], i_resam) += DIRECT_A1D_ELEM(thr_wsum_sigma2_noise[img_id], i); - DIRECT_A1D_ELEM(baseMLO->wsum_model.sumw_ctf2[optics_group], i_resam) += DIRECT_A1D_ELEM(thr_wsum_ctf2[img_id], i); + int my_image_size = baseMLO->mydata.getOpticsImageSize(optics_group); + RFLOAT my_pixel_size = baseMLO->mydata.getOpticsPixelSize(optics_group); + RFLOAT remap_image_sizes = + (baseMLO->mymodel.ori_size * baseMLO->mymodel.pixel_size) / (my_image_size * my_pixel_size); + FOR_ALL_DIRECT_ELEMENTS_IN_ARRAY1D(thr_wsum_sigma2_noise) { + int i_resam = ROUND(i * remap_image_sizes); + if (i_resam < XSIZE(baseMLO->wsum_model.sigma2_noise[optics_group])) { + DIRECT_A1D_ELEM(baseMLO->wsum_model.sigma2_noise[optics_group], i_resam) += DIRECT_A1D_ELEM( + thr_wsum_sigma2_noise, i); + DIRECT_A1D_ELEM(baseMLO->wsum_model.sumw_ctf2[optics_group], i_resam) += DIRECT_A1D_ELEM( + thr_wsum_ctf2, i); + + if (do_subtomo_correction) + DIRECT_A1D_ELEM(baseMLO->wsum_model.sumw_stMulti[optics_group], i_resam) += DIRECT_A1D_ELEM( + thr_wsum_stMulti, i); + } + } + baseMLO->wsum_model.sumw_group[optics_group] += thr_sumw_group; + if (baseMLO->do_scale_correction) { + baseMLO->wsum_model.wsum_signal_product[igroup] += thr_wsum_signal_product_spectra; + baseMLO->wsum_model.wsum_reference_power[igroup] += thr_wsum_reference_power_spectra; + } + for (int n = 0; n < baseMLO->mymodel.nr_classes; n++) { + baseMLO->wsum_model.pdf_class[n] += thr_wsum_pdf_class[n]; + if (baseMLO->mymodel.ref_dim == 2) { + XX(baseMLO->wsum_model.prior_offset_class[n]) += thr_wsum_prior_offsetx_class[n]; + YY(baseMLO->wsum_model.prior_offset_class[n]) += thr_wsum_prior_offsety_class[n]; + } + } - if (do_subtomo_correction) - DIRECT_A1D_ELEM(baseMLO->wsum_model.sumw_stMulti[optics_group], i_resam) += DIRECT_A1D_ELEM(thr_wsum_stMulti[img_id], i); - } - } - baseMLO->wsum_model.sumw_group[optics_group] += thr_sumw_group[img_id]; - if (baseMLO->do_scale_correction) - { - baseMLO->wsum_model.wsum_signal_product[igroup] += thr_wsum_signal_product_spectra[img_id]; - baseMLO->wsum_model.wsum_reference_power[igroup] += thr_wsum_reference_power_spectra[img_id]; - } - } - for (int n = 0; n < baseMLO->mymodel.nr_classes; n++) - { - baseMLO->wsum_model.pdf_class[n] += thr_wsum_pdf_class[n]; - if (baseMLO->mymodel.ref_dim == 2) - { - XX(baseMLO->wsum_model.prior_offset_class[n]) += thr_wsum_prior_offsetx_class[n]; - YY(baseMLO->wsum_model.prior_offset_class[n]) += thr_wsum_prior_offsety_class[n]; - } - } + for (int n = 0; n < baseMLO->mymodel.nr_classes * baseMLO->mymodel.nr_bodies; n++) { + if (!(baseMLO->do_skip_align || baseMLO->do_skip_rotate)) + baseMLO->wsum_model.pdf_direction[n] += thr_wsum_pdf_direction[n]; + } - for (int n = 0; n < baseMLO->mymodel.nr_classes * baseMLO->mymodel.nr_bodies; n++) - { - if (!(baseMLO->do_skip_align || baseMLO->do_skip_rotate) ) - baseMLO->wsum_model.pdf_direction[n] += thr_wsum_pdf_direction[n]; - } + baseMLO->wsum_model.sigma2_offset += thr_wsum_sigma2_offset; - baseMLO->wsum_model.sigma2_offset += thr_wsum_sigma2_offset; + if (baseMLO->do_norm_correction && baseMLO->mymodel.nr_bodies == 1) + baseMLO->wsum_model.avg_norm_correction += thr_avg_norm_correction; - if (baseMLO->do_norm_correction && baseMLO->mymodel.nr_bodies == 1) - baseMLO->wsum_model.avg_norm_correction += thr_avg_norm_correction; + baseMLO->wsum_model.LL += thr_sum_dLL; + baseMLO->wsum_model.ave_Pmax += thr_sum_Pmax; - baseMLO->wsum_model.LL += thr_sum_dLL; - baseMLO->wsum_model.ave_Pmax += thr_sum_Pmax; - } + } // end pragma lock } // end if !do_skip_maximization CTOC(accMLO->timer,"store_post_gpu"); @@ -3395,7 +3344,7 @@ baseMLO->timer.toc(baseMLO->TIMING_ESP_DIFF2_D); CTIC(timer,"getAllSquaredDifferencesFine"); getAllSquaredDifferencesFine(ipass, op, sp, baseMLO, myInstance, FinePassWeights, FinePassClassMasks, FineProjectionData, ptrFactory, ibody, bundleD2); CTOC(timer,"getAllSquaredDifferencesFine"); - FinePassWeights[0].weights.cpToHost(); + FinePassWeights.weights.cpToHost(); AccPtr Mweight = ptrFactory.make(); //DUMMY diff --git a/src/acc/cuda/cuda_kernels/diff2.cuh b/src/acc/cuda/cuda_kernels/diff2.cuh index e876da823..e018f1427 100644 --- a/src/acc/cuda/cuda_kernels/diff2.cuh +++ b/src/acc/cuda/cuda_kernels/diff2.cuh @@ -324,9 +324,6 @@ __global__ void cuda_kernel_diff2_fine( if (tid < trans_num) { s_outs[tid]=s[tid*block_sz]+sum_init; - } - if (tid < trans_num) - { iy=d_job_idx[bid]+tid; g_diff2s[iy] += s_outs[tid]; } diff --git a/src/ml_optimiser.cpp b/src/ml_optimiser.cpp index 33e2bd34b..ab07609fb 100644 --- a/src/ml_optimiser.cpp +++ b/src/ml_optimiser.cpp @@ -3839,7 +3839,7 @@ void MlOptimiser::expectationSomeParticles(long int my_first_part_id, long int m FileName fn_img, fn_stack, fn_open_stack=""; // Store total number of particle images in this bunch of SomeParticles, and set translations and orientations for skip_align/rotate - long int my_metadata_offset = 0; + long int my_imagedata_offset = 0; exp_imgs.clear(); int metadata_offset = 0; for (long int part_id_sorted = my_first_part_id; part_id_sorted <= my_last_part_id; part_id_sorted++) @@ -3908,14 +3908,14 @@ void MlOptimiser::expectationSomeParticles(long int my_first_part_id, long int m { // Read in the actual image from disc, only open/close common stacks once // Read in all images, only open/close common stacks once - for (int img_id = 0; img_id < mydata.numberOfImagesInParticle(part_id); img_id++, my_metadata_offset++) + for (int img_id = 0; img_id < mydata.numberOfImagesInParticle(part_id); img_id++,my_imagedata_offset++) { // Get the filename if (!mydata.getImageNameOnScratch(part_id, img_id, fn_img)) { std::istringstream split(exp_fn_img); - for (int i = 0; i <= my_metadata_offset; i++) + for (int i = 0; i <= my_imagedata_offset; i++) { getline(split, fn_img); } @@ -4116,12 +4116,12 @@ void MlOptimiser::expectationOneParticle(long int part_id_sorted, int thread_id) // Here define all kind of local arrays that will be needed std::vector > exp_Fimg, exp_Fimg_nomask; std::vector > > exp_local_Fimgs_shifted, exp_local_Fimgs_shifted_nomask; - std::vector > exp_Fctf, exp_local_Fctf, exp_local_Minvsigma2,exp_STMulti; + std::vector > exp_Fctf, exp_local_Fctf, exp_local_Minvsigma2; std::vector exp_pointer_dir_nonzeroprior, exp_pointer_psi_nonzeroprior; std::vector exp_directions_prior, exp_psi_prior, exp_local_sqrtXi2; int exp_current_image_size, exp_current_oversampling; std::vector exp_highres_Xi2_img; - MultidimArray exp_Mweight; + MultidimArray exp_Mweight, exp_STMulti; MultidimArray exp_Mcoarse_significant; // And from storeWeightedSums RFLOAT exp_min_diff2, exp_sum_weight, exp_significant_weight, exp_max_weight; @@ -4148,11 +4148,6 @@ void MlOptimiser::expectationOneParticle(long int part_id_sorted, int thread_id) exp_Fimg_nomask.resize(my_nr_images); exp_Fctf.resize(my_nr_images); - if (mydata.is_3D) - { - exp_STMulti.resize(my_nr_images); - } - // Then calculate all Fourier Transform of masked and unmasked image and the CTF #ifdef TIMING if (part_id_sorted == exp_my_first_part_id) @@ -5498,7 +5493,7 @@ void MlOptimiser::getFourierTransformsAndCtfs( std::vector &exp_pointer_psi_nonzeroprior, std::vector &exp_directions_prior, std::vector &exp_psi_prior, - std::vector > &exp_STMulti) + MultidimArray &exp_STMulti) { Matrix2D Aori; @@ -5812,7 +5807,7 @@ void MlOptimiser::getFourierTransformsAndCtfs( // Read from disc FileName fn_img; std::istringstream split(exp_fn_img); - for (int i = 0; i <= my_metadata_offset; i++) + for (int i = 0; i <= my_imagedata_offset; i++) getline(split, fn_img); img.read(fn_img); @@ -5820,12 +5815,12 @@ void MlOptimiser::getFourierTransformsAndCtfs( // Check that this is the same as the image in exp_imgs vector Image diff; - if (my_metadata_offset >= exp_imgs.size()) + if (my_imagedata_offset >= exp_imgs.size()) { - std::cerr << " my_metadata_offset= " < 1e-6) { std::cerr << "metadata_offset= " < 0 ) + if ( NZYXSIZE(FstMulti) > 0 && img_id == 0) { applySubtomoCorrection(exp_Fimg[img_id], exp_Fimg_nomask[img_id], Fctf, FstMulti); - exp_STMulti[img_id] = FstMulti; + exp_STMulti = FstMulti; } + // Store Fctf exp_Fctf[img_id] = Fctf; @@ -6240,7 +6236,7 @@ void MlOptimiser::getFourierTransformsAndCtfs( { /* for (int j = 0; j < XSIZE(exp_metadata); j++) - std::cerr << " j= " << j << " DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, j)= " << DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, j) << std::endl; + std::cerr << " j= " << j << " DIRECT_A2D_ELEM(exp_metadata, metadata_offset, j)= " << DIRECT_A2D_ELEM(exp_metadata, metadata_offset, j) << std::endl; Matrix2D B; B = (mymodel.orient_bodies[obody]).transpose() * Aresi * mymodel.orient_bodies[obody]; std::cerr << " B= " << B << std::endl; @@ -6248,11 +6244,11 @@ void MlOptimiser::getFourierTransformsAndCtfs( std::cerr << " mymodel.orient_bodies[obody]= " << mymodel.orient_bodies[obody] << std::endl; std::cerr << " Aori= " << Aori << std::endl; std::cerr << " Abody= " << Abody << std::endl; - std::cerr << " obody= " << obody+1 << "ocol_rot= " << DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, ocol_rot) - << " obody= " << obody+1 << "ocol_tilt= " << DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, ocol_tilt) - << " obody= " << obody+1 << "ocol_psi= " << DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, ocol_psi) - << " ocol_xoff= " << DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, ocol_xoff) - << " ocol_yoff= " << DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, ocol_yoff) << std::endl; + std::cerr << " obody= " << obody+1 << "ocol_rot= " << DIRECT_A2D_ELEM(exp_metadata, mmetadata_offset, ocol_rot) + << " obody= " << obody+1 << "ocol_tilt= " << DIRECT_A2D_ELEM(exp_metadata, metadata_offset, ocol_tilt) + << " obody= " << obody+1 << "ocol_psi= " << DIRECT_A2D_ELEM(exp_metadata, metadata_offset, ocol_psi) + << " ocol_xoff= " << DIRECT_A2D_ELEM(exp_metadata, metadata_offset, ocol_xoff) + << " ocol_yoff= " << DIRECT_A2D_ELEM(exp_metadata, metadata_offset, ocol_yoff) << std::endl; */ windowFourierTransform(FTo, Faux, mymodel.ori_size); transformer.inverseFourierTransform(Faux, img()); @@ -6280,8 +6276,8 @@ void MlOptimiser::getFourierTransformsAndCtfs( #ifdef DEBUG_BODIES if (part_id == ROUND(debug1)) std::cerr << " obody: " << obody+1 << " projected COM= " << other_projected_com.transpose() << std::endl; - std::cerr << " obody: " << obody+1 << " refined (x,y)= " << DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, ocol_xoff) - << " , " << DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, ocol_yoff) << std::endl; + std::cerr << " obody: " << obody+1 << " refined (x,y)= " << DIRECT_A2D_ELEM(exp_metadata, metadata_offset, ocol_xoff) + << " , " << DIRECT_A2D_ELEM(exp_metadata, metadata_offset, ocol_yoff) << std::endl; #endif // Subtract refined obody-displacement @@ -6420,8 +6416,8 @@ void MlOptimiser::precalculateShiftedImagesCtfsAndInvSigma2s(bool do_also_unmask std::vector >&exp_local_Fctf, std::vector &exp_local_sqrtXi2, std::vector >&exp_local_Minvsigma2, - std::vector > &exp_STMulti, - std::vector > &exp_local_STMulti) + MultidimArray &exp_STMulti, + MultidimArray &exp_local_STMulti) { #ifdef TIMING @@ -6448,7 +6444,7 @@ void MlOptimiser::precalculateShiftedImagesCtfsAndInvSigma2s(bool do_also_unmask exp_local_Fctf.resize(exp_nr_images); exp_local_sqrtXi2.resize(exp_nr_images); - bool do_subtomo_correction = exp_STMulti.size() > 0 && NZYXSIZE(exp_STMulti[0]) > 0; + bool do_subtomo_correction = NZYXSIZE(exp_STMulti) > 0; MultidimArray Fimg, Fimg_nomask; for (int img_id = 0, my_trans_image = 0; img_id < exp_nr_images; img_id++) @@ -6538,12 +6534,12 @@ void MlOptimiser::precalculateShiftedImagesCtfsAndInvSigma2s(bool do_also_unmask if (do_subtomo_correction) { MultidimArray STmult; - windowFourierTransform(exp_STMulti[img_id], STmult, exp_current_image_size); + windowFourierTransform(exp_STMulti, STmult, exp_current_image_size); if (is_for_store_wsums) { // We store the downsized subtomogram Fourier Multiplicity weights for updates of sigma2_noise in the storeWeightedSums function - exp_local_STMulti[img_id] = STmult; + exp_local_STMulti = STmult; // We also undo the division by STmult in the first pass for getAllSquareDifferences, if in this pass do_ctf_invsig is false if (!do_ctf_invsig) @@ -6750,7 +6746,7 @@ void MlOptimiser::getAllSquaredDifferences(long int part_id, int ibody, std::vector > &exp_local_Minvsigma2, std::vector > &exp_local_Fctf, std::vector &exp_local_sqrtXi2, - std::vector > &exp_STMulti) + MultidimArray &exp_STMulti) { #ifdef TIMING @@ -6784,12 +6780,11 @@ void MlOptimiser::getAllSquaredDifferences(long int part_id, int ibody, if (exp_ipass==0) exp_Mcoarse_significant.clear(); - exp_min_diff2.clear(); - exp_min_diff2.resize(exp_nr_images, LARGE_NUMBER); + exp_min_diff2 = LARGE_NUMBER; std::vector > dummy; std::vector > > dummy2; - std::vector > dymmyR; + MultidimArray dymmyR; precalculateShiftedImagesCtfsAndInvSigma2s(false, false, part_id, exp_current_oversampling, metadata_offset, exp_itrans_min, exp_itrans_max, exp_Fimg, dummy, exp_Fctf, exp_local_Fimgs_shifted, dummy2, @@ -7862,9 +7857,9 @@ void MlOptimiser::storeWeightedSums(long int part_id, int ibody, Matrix1D &exp_prior, MultidimArray &exp_Mweight, MultidimArray &exp_Mcoarse_significant, - std::vector &exp_significant_weight, - std::vector &exp_sum_weight, - std::vector &exp_max_weight, + RFLOAT &exp_significant_weight, + RFLOAT &exp_sum_weight, + RFLOAT &exp_max_weight, std::vector &exp_pointer_dir_nonzeroprior, std::vector &exp_pointer_psi_nonzeroprior, std::vector &exp_directions_prior, std::vector &exp_psi_prior, std::vector > > &exp_local_Fimgs_shifted, @@ -7872,7 +7867,7 @@ void MlOptimiser::storeWeightedSums(long int part_id, int ibody, std::vector > &exp_local_Minvsigma2, std::vector > &exp_local_Fctf, std::vector &exp_local_sqrtXi2, - std::vector > &exp_STMulti) + MultidimArray &exp_STMulti) { #ifdef TIMING if (part_id == mydata.sorted_idx[exp_my_first_part_id]) @@ -7886,10 +7881,15 @@ void MlOptimiser::storeWeightedSums(long int part_id, int ibody, long int exp_nr_oversampled_rot = sampling.oversamplingFactorOrientations(exp_current_oversampling); long int exp_nr_oversampled_trans = sampling.oversamplingFactorTranslations(exp_current_oversampling); - std::vector > exp_local_STMulti; - bool do_subtomo_correction = exp_STMulti.size() > 0 && NZYXSIZE(exp_STMulti[0]) > 0; - if (do_subtomo_correction) - exp_local_STMulti.resize(exp_nr_images); + // Assuming one group_id and optics_group for all images in this particle.... + int group_id = mydata.getGroupId(part_id, 0); + const int optics_group = mydata.getOpticsGroup(part_id, 0); + RFLOAT my_pixel_size = mydata.getImagePixelSize(part_id, 0); + int my_image_size = mydata.getOpticsImageSize(optics_group); + bool ctf_premultiplied = mydata.obsModel.getCtfPremultiplied(optics_group); + + MultidimArray exp_local_STMulti; + bool do_subtomo_correction = NZYXSIZE(exp_STMulti) > 0; // Re-do below because now also want unmasked images AND if (stricht_highres_exp >0.) then may need to resize precalculateShiftedImagesCtfsAndInvSigma2s(true, true, part_id, exp_current_oversampling, metadata_offset, @@ -7900,53 +7900,24 @@ void MlOptimiser::storeWeightedSums(long int part_id, int ibody, // Set those back here for (int img_id = 0; img_id < exp_nr_images; img_id++) { - int optics_group = mydata.getOpticsGroup(part_id, img_id); DIRECT_MULTIDIM_ELEM(exp_local_Minvsigma2[img_id], 0) = 1. / (sigma2_fudge * DIRECT_A1D_ELEM(mymodel.sigma2_noise[optics_group], 0)); } // Initialise the maximum of all weights to a negative value - exp_max_weight.clear(); - exp_max_weight.resize(exp_nr_images, -1.); + exp_max_weight = -1.; // For norm_correction and scale_correction of this particle - std::vector exp_wsum_norm_correction; - std::vector exp_wsum_scale_correction_XA, exp_wsum_scale_correction_AA; - std::vector thr_wsum_signal_product_spectra, thr_wsum_reference_power_spectra; - exp_wsum_norm_correction.resize(exp_nr_images, 0.); - - // For scale_correction - if (do_scale_correction) - { - exp_wsum_scale_correction_XA.resize(exp_nr_images); - exp_wsum_scale_correction_AA.resize(exp_nr_images); - thr_wsum_signal_product_spectra.resize(exp_nr_images); - thr_wsum_reference_power_spectra.resize(exp_nr_images); - } + RFLOAT exp_wsum_norm_correction = 0.; + RFLOAT exp_wsum_scale_correction_XA = 0., exp_wsum_scale_correction_AA = 0.; + RFLOAT thr_wsum_signal_product_spectra = 0., thr_wsum_reference_power_spectra = 0.; //Sigma2_noise estimation - std::vector > thr_wsum_sigma2_noise, thr_wsum_ctf2; - std::vector > thr_wsum_stMulti; - // Wsum_sigma_noise2 is a 1D-spectrum for each img_id - thr_wsum_sigma2_noise.resize(exp_nr_images); - thr_wsum_ctf2.resize(exp_nr_images); - if (do_subtomo_correction) - thr_wsum_stMulti.resize(exp_nr_images); - - for (int img_id = 0; img_id < exp_nr_images; img_id++) - { - int optics_group = mydata.getOpticsGroup(part_id, img_id); - thr_wsum_sigma2_noise[img_id].initZeros(image_full_size[optics_group]/2 + 1); - thr_wsum_ctf2[img_id].initZeros(image_full_size[optics_group]/2 + 1); - if (do_subtomo_correction) - thr_wsum_stMulti[img_id].initZeros(image_full_size[optics_group]/2 + 1); - if (do_scale_correction) - { - exp_wsum_scale_correction_XA[img_id] = 0.; - exp_wsum_scale_correction_AA[img_id] = 0.; - thr_wsum_signal_product_spectra[img_id] = 0.; - thr_wsum_reference_power_spectra[img_id] = 0.; - } - } + MultidimArray thr_wsum_sigma2_noise, thr_wsum_ctf2, thr_wsum_stMulti; + mydata.getOpticsGroup(part_id, 0); + thr_wsum_sigma2_noise.initZeros(image_full_size[optics_group]/2 + 1); + thr_wsum_ctf2.initZeros(image_full_size[optics_group]/2 + 1); + if (do_subtomo_correction) + thr_wsum_stMulti.initZeros(image_full_size[optics_group]/2 + 1); std::vector< RFLOAT> oversampled_rot, oversampled_tilt, oversampled_psi; std::vector oversampled_translations_x, oversampled_translations_y, oversampled_translations_z; @@ -7988,14 +7959,13 @@ void MlOptimiser::storeWeightedSums(long int part_id, int ibody, // Make local copies of weighted sums (except BPrefs, which are too big) // so that there are not too many mutex locks below std::vector > thr_wsum_pdf_direction; - std::vector thr_sumw_group, thr_wsum_pdf_class, thr_wsum_prior_offsetx_class, thr_wsum_prior_offsety_class; + RFLOAT thr_sumw_group =0.; + std::vector thr_wsum_pdf_class, thr_wsum_prior_offsetx_class, thr_wsum_prior_offsety_class; RFLOAT thr_wsum_sigma2_offset; MultidimArray thr_metadata, zeroArray; // wsum_pdf_direction is a 1D-array (of length sampling.NrDirections()) for each class zeroArray.initZeros(sampling.NrDirections()); thr_wsum_pdf_direction.resize(mymodel.nr_classes * mymodel.nr_bodies, zeroArray); - // sumw_group is a RFLOAT for each group - thr_sumw_group.resize(exp_nr_images, 0.); // wsum_pdf_class is a RFLOAT for each class thr_wsum_pdf_class.resize(mymodel.nr_classes, 0.); if (mymodel.ref_dim == 2) @@ -8027,11 +7997,6 @@ void MlOptimiser::storeWeightedSums(long int part_id, int ibody, // The order of the looping here has changed for 3.1: different img_id have different optics_group and therefore different applyAnisoMag.... for (int img_id = 0; img_id < exp_nr_images; img_id++) { - int group_id = mydata.getGroupId(part_id, img_id); - const int optics_group = mydata.getOpticsGroup(part_id, img_id); - int my_metadata_offset = metadata_offset + img_id; - RFLOAT my_pixel_size = mydata.getImagePixelSize(part_id, img_id); - bool ctf_premultiplied = mydata.obsModel.getCtfPremultiplied(optics_group); // Loop over all oversampled orientations (only a single one in the first pass) for (long int iover_rot = 0; iover_rot < exp_nr_oversampled_rot; iover_rot++) @@ -8086,8 +8051,8 @@ void MlOptimiser::storeWeightedSums(long int part_id, int ibody, // This is an attempt to speed up illogically slow updates of wsum_sigma2_offset.... // It seems to make a big difference! RFLOAT myprior_x, myprior_y, myprior_z, old_offset_z; - RFLOAT old_offset_x = XX(exp_old_offset[img_id]); - RFLOAT old_offset_y = YY(exp_old_offset[img_id]); + RFLOAT old_offset_x = XX(exp_old_offset); + RFLOAT old_offset_y = YY(exp_old_offset); if (mymodel.ref_dim == 2 && mymodel.nr_bodies == 1) { myprior_x = XX(mymodel.prior_offset_class[exp_iclass]); @@ -8095,12 +8060,12 @@ void MlOptimiser::storeWeightedSums(long int part_id, int ibody, } else { - myprior_x = XX(exp_prior[img_id]); - myprior_y = YY(exp_prior[img_id]); + myprior_x = XX(exp_prior); + myprior_y = YY(exp_prior); if (mymodel.data_dim == 3) { - myprior_z = ZZ(exp_prior[img_id]); - old_offset_z = ZZ(exp_old_offset[img_id]); + myprior_z = ZZ(exp_prior); + old_offset_z = ZZ(exp_old_offset); } } @@ -8170,12 +8135,12 @@ void MlOptimiser::storeWeightedSums(long int part_id, int ibody, // Only deal with this sampling point if its weight was significant long int ihidden_over = ihidden * exp_nr_oversampled_trans * exp_nr_oversampled_rot + iover_rot * exp_nr_oversampled_trans + iover_trans; - RFLOAT weight = DIRECT_A2D_ELEM(exp_Mweight, img_id, ihidden_over); + RFLOAT weight = DIRECT_A1D_ELEM(exp_Mweight, ihidden_over); // Only sum weights for non-zero weights - if (weight >= exp_significant_weight[img_id]) + if (weight >= exp_significant_weight) { // Normalise the weight (do this after the comparison with exp_significant_weight!) - weight /= exp_sum_weight[img_id]; + weight /= exp_sum_weight; if (!do_skip_maximization) { @@ -8207,9 +8172,9 @@ void MlOptimiser::storeWeightedSums(long int part_id, int ibody, if (mymodel.data_dim == 3) zshift = oversampled_translations_z[iover_trans]; - RFLOAT rot_deg = DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_ROT); - RFLOAT tilt_deg = DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_TILT); - RFLOAT psi_deg = DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_PSI); + RFLOAT rot_deg = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ROT); + RFLOAT tilt_deg = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_TILT); + RFLOAT psi_deg = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PSI); transformCartesianAndHelicalCoords( xshift, yshift, zshift, xshift, yshift, zshift, @@ -8279,18 +8244,18 @@ void MlOptimiser::storeWeightedSums(long int part_id, int ibody, RFLOAT diff_imag = (DIRECT_MULTIDIM_ELEM(Frefctf, n)).imag - (*(Fimg_shift + n)).imag; RFLOAT wdiff2 = weight * (diff_real*diff_real + diff_imag*diff_imag); // group-wise sigma2_noise - DIRECT_MULTIDIM_ELEM(thr_wsum_sigma2_noise[img_id], ires) += wdiff2; + DIRECT_MULTIDIM_ELEM(thr_wsum_sigma2_noise, ires) += wdiff2; // For norm_correction - exp_wsum_norm_correction[img_id] += wdiff2; + exp_wsum_norm_correction += wdiff2; if (do_scale_correction && DIRECT_A1D_ELEM(mymodel.data_vs_prior_class[exp_iclass], ires) > 3.) { RFLOAT sumXA, sumA2; sumXA = (DIRECT_MULTIDIM_ELEM(Frefctf, n)).real * (*(Fimg_shift + n)).real; sumXA += (DIRECT_MULTIDIM_ELEM(Frefctf, n)).imag * (*(Fimg_shift + n)).imag; - exp_wsum_scale_correction_XA[img_id] += weight * sumXA; + exp_wsum_scale_correction_XA += weight * sumXA; sumA2 = (DIRECT_MULTIDIM_ELEM(Frefctf, n)).real * (DIRECT_MULTIDIM_ELEM(Frefctf, n)).real; sumA2 += (DIRECT_MULTIDIM_ELEM(Frefctf, n)).imag * (DIRECT_MULTIDIM_ELEM(Frefctf, n)).imag; - exp_wsum_scale_correction_AA[img_id] += weight * sumA2; + exp_wsum_scale_correction_AA += weight * sumA2; } } } @@ -8304,7 +8269,7 @@ void MlOptimiser::storeWeightedSums(long int part_id, int ibody, #endif // Store sum of weights for this group - thr_sumw_group[img_id] += weight; + thr_sumw_group += weight; // Store weights for this class and orientation thr_wsum_pdf_class[exp_iclass] += weight; @@ -8322,9 +8287,9 @@ void MlOptimiser::storeWeightedSums(long int part_id, int ibody, // If it is doing helical refinement AND Cartesian vector myprior has a length > 0, transform the vector to its helical coordinates if ( (do_helical_refine) && (!ignore_helical_symmetry) && (mypriors_len2 > 0.00001) ) { - RFLOAT rot_deg = DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_ROT); - RFLOAT tilt_deg = DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_TILT); - RFLOAT psi_deg = DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_PSI); + RFLOAT rot_deg = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ROT); + RFLOAT tilt_deg = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_TILT); + RFLOAT psi_deg = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PSI); transformCartesianAndHelicalCoords(myprior_x, myprior_y, myprior_z, myprior_x, myprior_y, myprior_z, rot_deg, tilt_deg, psi_deg, mymodel.data_dim, CART_TO_HELICAL_COORDS); } @@ -8451,10 +8416,11 @@ void MlOptimiser::storeWeightedSums(long int part_id, int ibody, } // end if !do_skip_maximization // Keep track of max_weight and the corresponding optimal hidden variables - if (weight > exp_max_weight[img_id]) + // SHWS7July2022: only do this for the first img_id + if (img_id == 0 && weight > exp_max_weight) { // Store optimal image parameters - exp_max_weight[img_id] = weight; + exp_max_weight = weight; //This is not necessary as rot, tilt and psi remain unchanged! //Euler_matrix2angles(A, rot, tilt, psi); @@ -8466,20 +8432,20 @@ void MlOptimiser::storeWeightedSums(long int part_id, int ibody, int icol_yoff = (mymodel.nr_bodies == 1) ? METADATA_YOFF : 4 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; int icol_zoff = (mymodel.nr_bodies == 1) ? METADATA_ZOFF : 5 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; - RFLOAT old_rot = DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, icol_rot); - DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, icol_rot) = rot; - RFLOAT old_tilt = DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, icol_tilt); - DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, icol_tilt) = tilt; - RFLOAT old_psi = DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, icol_psi); - DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, icol_psi) = psi; + RFLOAT old_rot = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_rot); + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_rot) = rot; + RFLOAT old_tilt = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_tilt); + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_tilt) = tilt; + RFLOAT old_psi = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_psi); + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_psi) = psi; Matrix1D shifts(mymodel.data_dim); // include old_offsets for normal refinement (i.e. non multi-body) - XX(shifts) = XX(exp_old_offset[img_id]) + oversampled_translations_x[iover_trans]; - YY(shifts) = YY(exp_old_offset[img_id]) + oversampled_translations_y[iover_trans]; + XX(shifts) = XX(exp_old_offset) + oversampled_translations_x[iover_trans]; + YY(shifts) = YY(exp_old_offset) + oversampled_translations_y[iover_trans]; if (mymodel.data_dim == 3) { - ZZ(shifts) = ZZ(exp_old_offset[img_id]) + oversampled_translations_z[iover_trans]; + ZZ(shifts) = ZZ(exp_old_offset) + oversampled_translations_z[iover_trans]; } #ifdef DEBUG_BODIES2 std::cerr << ihidden_over << " weight= " << weight; @@ -8537,15 +8503,15 @@ void MlOptimiser::storeWeightedSums(long int part_id, int ibody, #endif } - DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, icol_xoff) = XX(shifts); - DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, icol_yoff) = YY(shifts); + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_xoff) = XX(shifts); + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_yoff) = YY(shifts); if (mymodel.data_dim == 3) - DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, icol_zoff) = ZZ(shifts); + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_zoff) = ZZ(shifts); if (ibody == 0) { - DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_CLASS) = (RFLOAT)exp_iclass + 1; - DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_PMAX) = exp_max_weight[img_id]; + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_CLASS) = (RFLOAT)exp_iclass + 1; + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PMAX) = exp_max_weight; } } // end if weight > exp_max_weight[img_id] } // end if weight >= exp_significant_weight @@ -8622,147 +8588,137 @@ void MlOptimiser::storeWeightedSums(long int part_id, int ibody, // loop over all images inside this particle for (int img_id = 0; img_id < exp_nr_images; img_id++) { - int group_id = mydata.getGroupId(part_id, img_id); - int my_metadata_offset = metadata_offset + img_id; - int optics_group = mydata.getOpticsGroup(part_id, img_id); - RFLOAT my_pixel_size = mydata.getOpticsPixelSize(optics_group); - int my_image_size = mydata.getOpticsImageSize(optics_group); - // If the current images were smaller than the original size, fill the rest of wsum_model.sigma2_noise with the power_class spectrum of the images for (int ires = image_current_size[optics_group]/2 + 1; ires < image_full_size[optics_group]/2 + 1; ires++) { - DIRECT_A1D_ELEM(thr_wsum_sigma2_noise[img_id], ires) += DIRECT_A1D_ELEM(exp_power_img[img_id], ires); + DIRECT_A1D_ELEM(thr_wsum_sigma2_noise, ires) += DIRECT_A1D_ELEM(exp_power_img[img_id], ires); // Also extend the weighted sum of the norm_correction - exp_wsum_norm_correction[img_id] += DIRECT_A1D_ELEM(exp_power_img[img_id], ires); + exp_wsum_norm_correction += DIRECT_A1D_ELEM(exp_power_img[img_id], ires); } + } - // Store norm_correction - // Multiply by old value because the old norm_correction term was already applied to the image - // Don't do this for multi-body refinement, where one always uses the norm_correction from the consensus refinement - if (do_norm_correction && mymodel.nr_bodies == 1) - { - RFLOAT old_norm_correction = DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_NORM); - old_norm_correction /= mymodel.avg_norm_correction; - // The factor two below is because exp_wsum_norm_correctiom is similar to sigma2_noise, which is the variance for the real/imag components - // The variance of the total image (on which one normalizes) is twice this value! - RFLOAT normcorr = old_norm_correction * sqrt(exp_wsum_norm_correction[img_id] * 2.); - thr_avg_norm_correction += normcorr; - // Now set the new norm_correction in the relevant position of exp_metadata - DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_NORM) = normcorr; - - // Print warning for strange norm-correction values - if (!((iter == 1 && do_firstiter_cc) || do_always_cc) && DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_NORM) > 10.) - { - std::cout << " WARNING: norm_correction= "<< DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_NORM) - << " for particle " << part_id << " in group " << group_id + 1 - << "; Are your groups large enough? Or is the reference on the correct greyscale?" << std::endl; - } - - } + // Store norm_correction + // Multiply by old value because the old norm_correction term was already applied to the image + // Don't do this for multi-body refinement, where one always uses the norm_correction from the consensus refinement + if (do_norm_correction && mymodel.nr_bodies == 1) + { + RFLOAT old_norm_correction = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_NORM); + old_norm_correction /= mymodel.avg_norm_correction; + // The factor two below is because exp_wsum_norm_correctiom is similar to sigma2_noise, which is the variance for the real/imag components + // The variance of the total image (on which one normalizes) is twice this value! + RFLOAT normcorr = old_norm_correction * sqrt(exp_wsum_norm_correction * 2.); + thr_avg_norm_correction += normcorr; + // Now set the new norm_correction in the relevant position of exp_metadata + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_NORM) = normcorr; + + // Print warning for strange norm-correction values + if (!((iter == 1 && do_firstiter_cc) || do_always_cc) && DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_NORM) > 10.) + { + std::cout << " WARNING: norm_correction= "<< DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_NORM) + << " for particle " << part_id << " in group " << group_id + 1 + << "; Are your groups large enough? Or is the reference on the correct greyscale?" << std::endl; + } - // Store weighted sums for scale_correction - if (do_scale_correction) - { - // Divide XA by the old scale_correction and AA by the square of that, because was incorporated into Fctf - exp_wsum_scale_correction_XA[img_id] /= mymodel.scale_correction[group_id]; - exp_wsum_scale_correction_AA [img_id]/= mymodel.scale_correction[group_id] * mymodel.scale_correction[group_id]; + } - thr_wsum_signal_product_spectra[img_id] += exp_wsum_scale_correction_XA[img_id]; - thr_wsum_reference_power_spectra[img_id] += exp_wsum_scale_correction_AA[img_id]; - } + // Store weighted sums for scale_correction + if (do_scale_correction) + { + // Divide XA by the old scale_correction and AA by the square of that, because was incorporated into Fctf + exp_wsum_scale_correction_XA /= mymodel.scale_correction[group_id]; + exp_wsum_scale_correction_AA/= mymodel.scale_correction[group_id] * mymodel.scale_correction[group_id]; - // Calculate DLL for each particle - RFLOAT logsigma2 = 0.; - RFLOAT remap_image_sizes = (mymodel.ori_size * mymodel.pixel_size) / (my_image_size * my_pixel_size); - FOR_ALL_DIRECT_ELEMENTS_IN_MULTIDIMARRAY(Mresol_fine[optics_group]) - { - int ires = DIRECT_MULTIDIM_ELEM(Mresol_fine[optics_group], n); - int ires_remapped = ROUND(remap_image_sizes * ires); - // Note there is no sqrt in the normalisation term because of the 2-dimensionality of the complex-plane - // Also exclude origin from logsigma2, as this will not be considered in the P-calculations - if (ires > 0 && ires_remapped < XSIZE(mymodel.sigma2_noise[optics_group])) - logsigma2 += log( 2. * PI * DIRECT_A1D_ELEM(mymodel.sigma2_noise[optics_group], ires_remapped)); - } - if (exp_sum_weight[img_id]==0) - { - std::cerr << " part_id= " << part_id << std::endl; - std::cerr << " img_id= " << img_id << std::endl; - std::cerr << " exp_min_diff2[img_id]= " << exp_min_diff2[img_id]<< std::endl; - std::cerr << " logsigma2= " << logsigma2 << std::endl; - int group_id = mydata.getGroupId(part_id, img_id); - std::cerr << " group_id= " << group_id << std::endl; - std::cerr << " ml_model.scale_correction[group_id]= " << mymodel.scale_correction[group_id] << std::endl; - std::cerr << " exp_significant_weight[img_id]= " << exp_significant_weight[img_id] << std::endl; - std::cerr << " exp_max_weight[img_id]= " << exp_max_weight[img_id] << std::endl; - std::cerr << " ml_model.sigma2_noise[optics_group]= " << mymodel.sigma2_noise[optics_group] << std::endl; - REPORT_ERROR("ERROR: exp_sum_weight[img_id]==0"); - } - RFLOAT dLL; - if ((iter==1 && do_firstiter_cc) || do_always_cc) - dLL = -exp_min_diff2[img_id]; - else - dLL = log(exp_sum_weight[img_id]) - exp_min_diff2[img_id] - logsigma2; + thr_wsum_signal_product_spectra += exp_wsum_scale_correction_XA; + thr_wsum_reference_power_spectra += exp_wsum_scale_correction_AA; + } - // Store dLL of each image in the output array, and keep track of total sum - DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_DLL) = dLL; - thr_sum_dLL += dLL; + // Calculate DLL for each particle + RFLOAT logsigma2 = 0.; + RFLOAT remap_image_sizes = (mymodel.ori_size * mymodel.pixel_size) / (my_image_size * my_pixel_size); + FOR_ALL_DIRECT_ELEMENTS_IN_MULTIDIMARRAY(Mresol_fine[optics_group]) + { + int ires = DIRECT_MULTIDIM_ELEM(Mresol_fine[optics_group], n); + int ires_remapped = ROUND(remap_image_sizes * ires); + // Note there is no sqrt in the normalisation term because of the 2-dimensionality of the complex-plane + // Also exclude origin from logsigma2, as this will not be considered in the P-calculations + if (ires > 0 && ires_remapped < XSIZE(mymodel.sigma2_noise[optics_group])) + logsigma2 += log( 2. * PI * DIRECT_A1D_ELEM(mymodel.sigma2_noise[optics_group], ires_remapped)); + } + if (exp_sum_weight==0) + { + std::cerr << " part_id= " << part_id << std::endl; + std::cerr << " exp_min_diff2= " << exp_min_diff2<< std::endl; + std::cerr << " logsigma2= " << logsigma2 << std::endl; + int group_id = mydata.getGroupId(part_id, 0); + std::cerr << " group_id= " << group_id << std::endl; + std::cerr << " ml_model.scale_correction[group_id]= " << mymodel.scale_correction[group_id] << std::endl; + std::cerr << " exp_significant_weight= " << exp_significant_weight << std::endl; + std::cerr << " exp_max_weight= " << exp_max_weight << std::endl; + std::cerr << " ml_model.sigma2_noise[optics_group]= " << mymodel.sigma2_noise[optics_group] << std::endl; + REPORT_ERROR("ERROR: exp_sum_weight==0"); + } + RFLOAT dLL; + if ((iter==1 && do_firstiter_cc) || do_always_cc) + dLL = -exp_min_diff2; + else + dLL = log(exp_sum_weight) - exp_min_diff2 - logsigma2; - // Also store sum of Pmax - thr_sum_Pmax += DIRECT_A2D_ELEM(exp_metadata, my_metadata_offset, METADATA_PMAX); + // Store dLL of each image in the output array, and keep track of total sum + DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_DLL) = dLL; + thr_sum_dLL += dLL; - } // end loop img_id + // Also store sum of Pmax + thr_sum_Pmax += DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PMAX); - // Now, inside a global_mutex, update the other weighted sums among all threads + // Now, inside a global_mutex, update the other weighted sums among all threads if (!do_skip_maximization) { omp_set_lock(&global_mutex); - for (int img_id = 0; img_id < exp_nr_images; img_id++) - { - long int igroup = mydata.getGroupId(part_id, img_id); - int optics_group = mydata.getOpticsGroup(part_id, img_id); - if (do_subtomo_correction) + long int igroup = mydata.getGroupId(part_id, 0); + int optics_group = mydata.getOpticsGroup(part_id, 0); + if (do_subtomo_correction) + { + FOR_ALL_DIRECT_ELEMENTS_IN_MULTIDIMARRAY(Mresol_fine[optics_group]) { - FOR_ALL_DIRECT_ELEMENTS_IN_MULTIDIMARRAY(Mresol_fine[optics_group]) - { - int ires = DIRECT_MULTIDIM_ELEM(Mresol_fine[optics_group], n); - if (ires > -1) - DIRECT_MULTIDIM_ELEM(thr_wsum_stMulti[img_id], ires) += DIRECT_MULTIDIM_ELEM(exp_local_STMulti[img_id], n); - } + int ires = DIRECT_MULTIDIM_ELEM(Mresol_fine[optics_group], n); + if (ires > -1) + DIRECT_MULTIDIM_ELEM(thr_wsum_stMulti, ires) += DIRECT_MULTIDIM_ELEM(exp_local_STMulti, n); } + } - if (mydata.obsModel.getCtfPremultiplied(optics_group)) + if (mydata.obsModel.getCtfPremultiplied(optics_group)) + { + RFLOAT myscale = XMIPP_MAX(0.001, mymodel.scale_correction[igroup]); + FOR_ALL_DIRECT_ELEMENTS_IN_MULTIDIMARRAY(Mresol_fine[optics_group]) { - RFLOAT myscale = XMIPP_MAX(0.001, mymodel.scale_correction[igroup]); - FOR_ALL_DIRECT_ELEMENTS_IN_MULTIDIMARRAY(Mresol_fine[optics_group]) - { - int ires = DIRECT_MULTIDIM_ELEM(Mresol_fine[optics_group], n); - if (ires > -1) - DIRECT_MULTIDIM_ELEM(thr_wsum_ctf2[img_id], ires) += myscale * DIRECT_MULTIDIM_ELEM(exp_local_Fctf[img_id], n); - } + int ires = DIRECT_MULTIDIM_ELEM(Mresol_fine[optics_group], n); + if (ires > -1) + DIRECT_MULTIDIM_ELEM(thr_wsum_ctf2, ires) += myscale * DIRECT_MULTIDIM_ELEM(exp_local_Fctf[0], n); } + } + + int my_image_size = mydata.getOpticsImageSize(optics_group); + RFLOAT my_pixel_size = mydata.getOpticsPixelSize(optics_group); + RFLOAT remap_image_sizes = (mymodel.ori_size * mymodel.pixel_size) / (my_image_size * my_pixel_size); + FOR_ALL_DIRECT_ELEMENTS_IN_ARRAY1D(thr_wsum_sigma2_noise) + { + int i_resam = ROUND(i * remap_image_sizes); + if (i_resam < XSIZE(wsum_model.sigma2_noise[optics_group])) + { + DIRECT_A1D_ELEM(wsum_model.sigma2_noise[optics_group], i_resam) += DIRECT_A1D_ELEM(thr_wsum_sigma2_noise, i); + DIRECT_A1D_ELEM(wsum_model.sumw_ctf2[optics_group], i_resam) += DIRECT_A1D_ELEM(thr_wsum_ctf2, i); + if (do_subtomo_correction) + DIRECT_A1D_ELEM(wsum_model.sumw_stMulti[optics_group], i_resam) += DIRECT_A1D_ELEM(thr_wsum_stMulti, i); + } + } + wsum_model.sumw_group[optics_group] += thr_sumw_group; + if (do_scale_correction) + { + wsum_model.wsum_signal_product[igroup] += thr_wsum_signal_product_spectra; + wsum_model.wsum_reference_power[igroup] += thr_wsum_reference_power_spectra; + } - int my_image_size = mydata.getOpticsImageSize(optics_group); - RFLOAT my_pixel_size = mydata.getOpticsPixelSize(optics_group); - RFLOAT remap_image_sizes = (mymodel.ori_size * mymodel.pixel_size) / (my_image_size * my_pixel_size); - FOR_ALL_DIRECT_ELEMENTS_IN_ARRAY1D(thr_wsum_sigma2_noise[img_id]) - { - int i_resam = ROUND(i * remap_image_sizes); - if (i_resam < XSIZE(wsum_model.sigma2_noise[optics_group])) - { - DIRECT_A1D_ELEM(wsum_model.sigma2_noise[optics_group], i_resam) += DIRECT_A1D_ELEM(thr_wsum_sigma2_noise[img_id], i); - DIRECT_A1D_ELEM(wsum_model.sumw_ctf2[optics_group], i_resam) += DIRECT_A1D_ELEM(thr_wsum_ctf2[img_id], i); - if (do_subtomo_correction) - DIRECT_A1D_ELEM(wsum_model.sumw_stMulti[optics_group], i_resam) += DIRECT_A1D_ELEM(thr_wsum_stMulti[img_id], i); - } - } - wsum_model.sumw_group[optics_group] += thr_sumw_group[img_id]; - if (do_scale_correction) - { - wsum_model.wsum_signal_product[igroup] += thr_wsum_signal_product_spectra[img_id]; - wsum_model.wsum_reference_power[igroup] += thr_wsum_reference_power_spectra[img_id]; - } - } for (int n = 0; n < mymodel.nr_classes; n++) { wsum_model.pdf_class[n] += thr_wsum_pdf_class[n]; diff --git a/src/ml_optimiser.h b/src/ml_optimiser.h index d4d8a636e..bba74f31c 100644 --- a/src/ml_optimiser.h +++ b/src/ml_optimiser.h @@ -1069,7 +1069,7 @@ class MlOptimiser std::vector &exp_pointer_psi_nonzeroprior, std::vector &exp_directions_prior, std::vector &exp_psi_prior, - std::vector > &exp_STweight); + MultidimArray &exp_STweight); /* Store all shifted FourierTransforms in a vector * also store precalculated 2D matrices with 1/sigma2_noise @@ -1085,8 +1085,8 @@ class MlOptimiser std::vector > &exp_local_Fctf, std::vector &exp_local_sqrtXi2, std::vector > &exp_local_Minvsigma2, - std::vector > &exp_STweight, - std::vector > &exp_local_STMulti); + MultidimArray &exp_STweight, + MultidimArray &exp_local_STMulti); // Given exp_Mcoarse_significant, check for iorient whether any of the particles has any significant (coarsely sampled) translation bool isSignificantAnyImageAnyTranslation(long int iorient, @@ -1109,7 +1109,7 @@ class MlOptimiser std::vector > &exp_local_Minvsigma2, std::vector > &exp_local_Fctf, std::vector &exp_local_sqrtXi, - std::vector > &exp_STweight); + MultidimArray &exp_STweight); // Convert all squared difference terms to weights. // Also calculates exp_sum_weight and, for adaptive approach, also exp_significant_weight @@ -1148,7 +1148,7 @@ class MlOptimiser std::vector > &exp_local_Minvsigma2, std::vector > &exp_local_Fctf, std::vector &exp_local_sqrtXi2, - std::vector > &exp_STweight); + MultidimArray &exp_STweight); /** Monitor the changes in the optimal translations, orientations and class assignments for some particles */ void monitorHiddenVariableChanges(long int my_first_part_id, long int my_last_part_id); From aa82b75eec65fec8fb42ca72042d5d04a6a7fbfd Mon Sep 17 00:00:00 2001 From: scheres Date: Thu, 7 Jul 2022 13:17:35 +0100 Subject: [PATCH 076/495] debugging GPU code; CAREFUL reverted += for loop over img_id back to = in ../src/acc/cuda/cuda_kernels/helper.cu and ../src/acc/cuda/cuda_kernels/diff2.cuh --- src/acc/acc_ml_optimiser_impl.h | 21 ++++++++++++++++++++- src/acc/cuda/cuda_kernels/diff2.cuh | 3 ++- src/acc/cuda/cuda_kernels/helper.cu | 5 +++-- src/ml_optimiser.cpp | 21 +++++++++++++++------ 4 files changed, 40 insertions(+), 10 deletions(-) diff --git a/src/acc/acc_ml_optimiser_impl.h b/src/acc/acc_ml_optimiser_impl.h index dfd408eca..28045b829 100644 --- a/src/acc/acc_ml_optimiser_impl.h +++ b/src/acc/acc_ml_optimiser_impl.h @@ -1159,6 +1159,16 @@ void getAllSquaredDifferencesCoarse( do_CC, accMLO->dataIs3D); + /* + Mweight.cpToHost(); + for (int i= 0; i < Mweight.getSize(); i++) + if (Mweight[i]>0.)std::cerr << " Mweight=" << Mweight[i] << std::endl; + allWeights.hostAlloc(); + allWeights.cpToHost(); + for (int i= 0; i < allWeights.getSize(); i++) + std::cerr << " allWweights-before =" << allWeights[i] << std::endl; + */ + mapAllWeightsToMweights( ~projectorPlans[iclass].iorientclasses, &(~allWeights)[allWeights_pos], @@ -1168,6 +1178,12 @@ void getAllSquaredDifferencesCoarse( accMLO->classStreams[iclass] ); + /* + allWeights.cpToHost(); + for (int i= 0; i < allWeights.getSize(); i++) + std::cerr << " allWweights-after =" << allWeights[i] << std::endl; + exit(1); + */ /*==================================== Retrieve Results ======================================*/ @@ -1811,6 +1827,7 @@ void convertAllSquaredDifferencesToWeights(unsigned exp_ipass, XFLOAT weights_max = AccUtilities::getMaxOnDevice(ipartMweight); + std::cerr <<" weights_max= " << weights_max << std::endl; /* * Add 50 since we want to stay away from e^88, which approaches the single precision limit. @@ -3287,7 +3304,9 @@ baseMLO->timer.toc(baseMLO->TIMING_ESP_DIFF2_B); Mweight.setSize(weightsPerPart); Mweight.setHostPtr(op.Mweight.data); Mweight.deviceAlloc(); - deviceInitValue(Mweight, -std::numeric_limits::max()); + //deviceInitValue(Mweight, -std::numeric_limits::max()); + // SHWS 7July2022: not entirely sure about how this works, but as I'm adding to the diff2 for loop over all img_id, this can no longer be a large negative value... + deviceInitValue(Mweight, 0.); Mweight.streamSync(); CTIC(timer,"getAllSquaredDifferencesCoarse"); diff --git a/src/acc/cuda/cuda_kernels/diff2.cuh b/src/acc/cuda/cuda_kernels/diff2.cuh index e018f1427..b21e5ff94 100644 --- a/src/acc/cuda/cuda_kernels/diff2.cuh +++ b/src/acc/cuda/cuda_kernels/diff2.cuh @@ -325,7 +325,8 @@ __global__ void cuda_kernel_diff2_fine( { s_outs[tid]=s[tid*block_sz]+sum_init; iy=d_job_idx[bid]+tid; - g_diff2s[iy] += s_outs[tid]; + // SHWS 7July2022: this needs changing to += + g_diff2s[iy] = s_outs[tid]; } } } diff --git a/src/acc/cuda/cuda_kernels/helper.cu b/src/acc/cuda/cuda_kernels/helper.cu index 8834f2ee8..cbb4ba35d 100644 --- a/src/acc/cuda/cuda_kernels/helper.cu +++ b/src/acc/cuda/cuda_kernels/helper.cu @@ -961,8 +961,9 @@ __global__ void cuda_kernel_allweights_to_mweights( ) { size_t idx = blockIdx.x * block_size + threadIdx.x; - if (idx < orientation_num*translation_num) - d_mweights[d_iorient[idx/translation_num] * translation_num + idx%translation_num] += + // SHWS 7July2022: this below needs changing to += + if (idx < orientation_num*translation_num) + d_mweights[d_iorient[idx/translation_num] * translation_num + idx%translation_num] = d_allweights[idx/translation_num * translation_num + idx%translation_num]; // TODO - isn't this just d_allweights[idx + idx%translation_num]? Really? } diff --git a/src/ml_optimiser.cpp b/src/ml_optimiser.cpp index ab07609fb..730d06df2 100644 --- a/src/ml_optimiser.cpp +++ b/src/ml_optimiser.cpp @@ -5498,6 +5498,7 @@ void MlOptimiser::getFourierTransformsAndCtfs( Matrix2D Aori; Matrix1D my_projected_com(mymodel.data_dim), my_refined_ibody_offset(mymodel.data_dim); + int exp_nr_images = mydata.numberOfImagesInParticle(part_id); // Get the norm_correction (for multi-body refinement: still use the one from the consensus refinement!) RFLOAT normcorr = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_NORM); @@ -5771,7 +5772,7 @@ void MlOptimiser::getFourierTransformsAndCtfs( FourierTransformer transformer; - for (int img_id = 0; img_id < mydata.numberOfImagesInParticle(part_id); img_id++) + for (int img_id = 0; img_id < exp_nr_images; img_id++) { Image img, rec_img; MultidimArray Fimg, Faux; @@ -6866,7 +6867,7 @@ void MlOptimiser::getAllSquaredDifferences(long int part_id, int ibody, { // loop over all images inside this particle - for (int img_id = 0; img_id < mydata.numberOfImagesInParticle(part_id); img_id++) + for (int img_id = 0; img_id < exp_nr_images; img_id++) { RFLOAT my_pixel_size = mydata.getImagePixelSize(part_id, img_id); @@ -7289,14 +7290,18 @@ void MlOptimiser::getAllSquaredDifferences(long int part_id, int ibody, // SHWS 6July2022: += instead of =, as summing over all imag_id.... - DIRECT_A1D_ELEM(exp_Mweight, ihidden_over) += diff2; + if (DIRECT_A1D_ELEM(exp_Mweight, ihidden_over) < 0.) + DIRECT_A1D_ELEM(exp_Mweight, ihidden_over) = diff2; + else + DIRECT_A1D_ELEM(exp_Mweight, ihidden_over) += diff2; // Keep track of minimum of all diff2, only for the last image in this series - if (img_id == mydata.numberOfImagesInParticle(part_id) -1 && diff2 < exp_min_diff2) + if (img_id == exp_nr_images-1 && diff2 < exp_min_diff2) { exp_min_diff2 = diff2; + /* - if (part_id == 0) + if (part_id == 0) { std::cerr << " part_id= " << part_id << " ihidden_over= " << ihidden_over << " diff2= " << diff2 << " x= " << oversampled_translations_x[iover_trans] << " y=" < Date: Thu, 7 Jul 2022 14:06:04 +0100 Subject: [PATCH 077/495] repaired bug in passing img_id*weightsPerPart to mapAllWeightsToMweights, also put back += in diff2.cuh and helper.cuh --- src/acc/acc_ml_optimiser_impl.h | 19 +------------------ src/acc/cuda/cuda_kernels/diff2.cuh | 2 +- src/acc/cuda/cuda_kernels/helper.cu | 2 +- 3 files changed, 3 insertions(+), 20 deletions(-) diff --git a/src/acc/acc_ml_optimiser_impl.h b/src/acc/acc_ml_optimiser_impl.h index 28045b829..63496deed 100644 --- a/src/acc/acc_ml_optimiser_impl.h +++ b/src/acc/acc_ml_optimiser_impl.h @@ -1159,31 +1159,15 @@ void getAllSquaredDifferencesCoarse( do_CC, accMLO->dataIs3D); - /* - Mweight.cpToHost(); - for (int i= 0; i < Mweight.getSize(); i++) - if (Mweight[i]>0.)std::cerr << " Mweight=" << Mweight[i] << std::endl; - allWeights.hostAlloc(); - allWeights.cpToHost(); - for (int i= 0; i < allWeights.getSize(); i++) - std::cerr << " allWweights-before =" << allWeights[i] << std::endl; - */ - mapAllWeightsToMweights( ~projectorPlans[iclass].iorientclasses, &(~allWeights)[allWeights_pos], - &(~Mweight)[weightsPerPart], + &(~Mweight)[0], projectorPlans[iclass].orientation_num, translation_num, accMLO->classStreams[iclass] ); - /* - allWeights.cpToHost(); - for (int i= 0; i < allWeights.getSize(); i++) - std::cerr << " allWweights-after =" << allWeights[i] << std::endl; - exit(1); - */ /*==================================== Retrieve Results ======================================*/ @@ -1827,7 +1811,6 @@ void convertAllSquaredDifferencesToWeights(unsigned exp_ipass, XFLOAT weights_max = AccUtilities::getMaxOnDevice(ipartMweight); - std::cerr <<" weights_max= " << weights_max << std::endl; /* * Add 50 since we want to stay away from e^88, which approaches the single precision limit. diff --git a/src/acc/cuda/cuda_kernels/diff2.cuh b/src/acc/cuda/cuda_kernels/diff2.cuh index b21e5ff94..46ec5854e 100644 --- a/src/acc/cuda/cuda_kernels/diff2.cuh +++ b/src/acc/cuda/cuda_kernels/diff2.cuh @@ -604,7 +604,7 @@ __global__ void cuda_kernel_diff2_CC_fine( if (tid < trans_num) { iy=d_job_idx[bid]+tid; - g_diff2s[iy] = s_outs[tid]; + g_diff2s[iy] += s_outs[tid]; } } } diff --git a/src/acc/cuda/cuda_kernels/helper.cu b/src/acc/cuda/cuda_kernels/helper.cu index cbb4ba35d..1cceb49b0 100644 --- a/src/acc/cuda/cuda_kernels/helper.cu +++ b/src/acc/cuda/cuda_kernels/helper.cu @@ -963,7 +963,7 @@ __global__ void cuda_kernel_allweights_to_mweights( size_t idx = blockIdx.x * block_size + threadIdx.x; // SHWS 7July2022: this below needs changing to += if (idx < orientation_num*translation_num) - d_mweights[d_iorient[idx/translation_num] * translation_num + idx%translation_num] = + d_mweights[d_iorient[idx/translation_num] * translation_num + idx%translation_num] += d_allweights[idx/translation_num * translation_num + idx%translation_num]; // TODO - isn't this just d_allweights[idx + idx%translation_num]? Really? } From 3da52d4fe4abc6de2f15d8dbbbca057327d8f325 Mon Sep 17 00:00:00 2001 From: scheres Date: Thu, 7 Jul 2022 14:11:50 +0100 Subject: [PATCH 078/495] set pixelSize to EMDL_TOMO_TILT_SERIES_PIXEL_SIZE not EMDL_MICROGRAPH_ORIGINAL_PIXEL_SIZE, --- src/jaz/tomography/tomogram_set.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/jaz/tomography/tomogram_set.cpp b/src/jaz/tomography/tomogram_set.cpp index adb9127a8..4545507b2 100644 --- a/src/jaz/tomography/tomogram_set.cpp +++ b/src/jaz/tomography/tomogram_set.cpp @@ -254,7 +254,7 @@ Tomogram TomogramSet::loadTomogram(int index, bool loadImageData) const double Q0; - globalTable.getValueSafely(EMDL_MICROGRAPH_ORIGINAL_PIXEL_SIZE, out.optics.pixelSize, index); + globalTable.getValueSafely(EMDL_TOMO_TILT_SERIES_PIXEL_SIZE, out.optics.pixelSize, index); globalTable.getValueSafely(EMDL_CTF_VOLTAGE, out.optics.voltage, index); globalTable.getValueSafely(EMDL_CTF_CS, out.optics.Cs, index); globalTable.getValueSafely(EMDL_CTF_Q0, Q0, index); @@ -418,7 +418,7 @@ void TomogramSet::addTomogramFromIMODStack( const CTF& ctf0 = ctfs[0]; - globalTable.setValue(EMDL_MICROGRAPH_ORIGINAL_PIXEL_SIZE, pixelSize, index); + globalTable.setValue(EMDL_TOMO_TILT_SERIES_PIXEL_SIZE, pixelSize, index); globalTable.setValue(EMDL_CTF_VOLTAGE, ctf0.kV, index); globalTable.setValue(EMDL_CTF_CS, ctf0.Cs, index); globalTable.setValue(EMDL_CTF_Q0, ctf0.Q0, index); @@ -633,14 +633,14 @@ void TomogramSet::generateSingleMetaDataTable(MetaDataTable &MDout, ObservationM // Store all the necessary optics stuff in an opticsGroup per tomogram RFLOAT pixelSize, voltage, Cs, Q0; std::string tomo_name = getTomogramName(t); - globalTable.getValueSafely(EMDL_MICROGRAPH_ORIGINAL_PIXEL_SIZE, pixelSize, t); + globalTable.getValueSafely(EMDL_TOMO_TILT_SERIES_PIXEL_SIZE, pixelSize, t); globalTable.getValueSafely(EMDL_CTF_VOLTAGE, voltage, t); globalTable.getValueSafely(EMDL_CTF_CS, Cs, t); globalTable.getValueSafely(EMDL_CTF_Q0, Q0, t); obsModel.opticsMdt.addObject(); obsModel.opticsMdt.setValue(EMDL_IMAGE_OPTICS_GROUP_NAME, tomo_name); obsModel.opticsMdt.setValue(EMDL_IMAGE_OPTICS_GROUP, t+1); - obsModel.opticsMdt.setValue(EMDL_MICROGRAPH_ORIGINAL_PIXEL_SIZE, pixelSize); + obsModel.opticsMdt.setValue(EMDL_TOMO_TILT_SERIES_PIXEL_SIZE, pixelSize); obsModel.opticsMdt.setValue(EMDL_CTF_VOLTAGE, voltage); obsModel.opticsMdt.setValue(EMDL_CTF_CS, Cs); obsModel.opticsMdt.setValue(EMDL_CTF_Q0, Q0); From 55ccc8f48fa6f28c251b0db3bd4bc6e071a9c92b Mon Sep 17 00:00:00 2001 From: dkimanius Date: Thu, 7 Jul 2022 14:43:57 +0100 Subject: [PATCH 079/495] Accelerated scalar add kernel implemented. Fix for static asserts in align_tiltseries_runner.cpp --- src/acc/acc_ml_optimiser_impl.h | 14 ++++++++++---- src/acc/cpu/cpu_kernels/helper.cpp | 20 ++++++++++++++++++-- src/acc/cpu/cpu_kernels/helper.h | 11 +++++++++++ src/acc/cuda/cuda_kernels/helper.cuh | 15 +++++++++++++++ src/acc/utilities.h | 19 +++++++++++++++++++ src/align_tiltseries_runner.cpp | 2 +- 6 files changed, 74 insertions(+), 7 deletions(-) diff --git a/src/acc/acc_ml_optimiser_impl.h b/src/acc/acc_ml_optimiser_impl.h index 63496deed..c0f62ac2c 100644 --- a/src/acc/acc_ml_optimiser_impl.h +++ b/src/acc/acc_ml_optimiser_impl.h @@ -1118,8 +1118,14 @@ void getAllSquaredDifferencesCoarse( buildCorrImage(baseMLO,op,corr_img,img_id,group_id); corr_img.cpToDevice(); - if (sp.nr_images > 1) REPORT_ERROR("TODO: Dari needs to write a function to add op.highres_Xi2_img[img_id] / 2. to all Weights, not initialise again!"); - deviceInitValue(allWeights, (XFLOAT) (op.highres_Xi2_img[img_id] / 2.)); + AccUtilities::add( + BLOCK_SIZE, + allWeights.getStream(), + (XFLOAT*)~allWeights, + (XFLOAT) (op.highres_Xi2_img[img_id] / 2.), + allWeights.getSize() + ); + allWeights_pos = 0; for (int exp_iclass = sp.iclass_min; exp_iclass <= sp.iclass_max; exp_iclass++) @@ -1674,8 +1680,8 @@ void convertAllSquaredDifferencesToWeights(unsigned exp_ipass, PassWeights.weights.cpToHost(); DEBUG_HANDLE_ERROR(cudaStreamSynchronize(cudaStreamPerThread)); #else - deviceInitValue(PassWeights[img_id].weights, (XFLOAT)0.0); - PassWeights[img_id].weights[min_pair.first] = (XFLOAT)1.0; + deviceInitValue(PassWeights.weights, (XFLOAT)0.0); + PassWeights.weights[min_pair.first] = (XFLOAT)1.0; #endif my_significant_weight = 0.999; diff --git a/src/acc/cpu/cpu_kernels/helper.cpp b/src/acc/cpu/cpu_kernels/helper.cpp index 02353cd54..066073453 100644 --- a/src/acc/cpu/cpu_kernels/helper.cpp +++ b/src/acc/cpu/cpu_kernels/helper.cpp @@ -806,6 +806,22 @@ void cpu_kernel_multi( T *A, for (size_t i = 0; i < image_size; i ++) OUT[i] = A[i]*B[i]*S; } + +template +void cpu_kernel_add( + T *A, + T S, + size_t size +) +{ +#ifdef DEBUG_CUDA + if (size < 0) + ACC_PTR_DEBUG_INFO("cpu_kernel_add: image_size < 0"); +#endif + for (size_t i = 0; i < size; i ++) + A[i] += S; +} + /* void batch_multi( int blockIdx_x, int blockIdx_y, @@ -1055,8 +1071,8 @@ template void CpuKernels::cpu_translate2D(XFLOAT *, template void CpuKernels::cpu_translate3D(XFLOAT *, XFLOAT *, size_t, int, int, int, int, int, int); -template void CpuKernels::cpu_kernel_multi( XFLOAT *, - XFLOAT, size_t); +template void CpuKernels::cpu_kernel_multi( XFLOAT *, XFLOAT, size_t); +template void CpuKernels::cpu_kernel_add( XFLOAT *, XFLOAT, size_t); template void CpuKernels::cpu_kernel_make_eulers_3D(int, int, XFLOAT *, XFLOAT *, XFLOAT *, XFLOAT *, unsigned long, XFLOAT *, XFLOAT *); diff --git a/src/acc/cpu/cpu_kernels/helper.h b/src/acc/cpu/cpu_kernels/helper.h index a3072b77a..186b34269 100644 --- a/src/acc/cpu/cpu_kernels/helper.h +++ b/src/acc/cpu/cpu_kernels/helper.h @@ -390,6 +390,17 @@ void cpu_kernel_multi( T *A, T *OUT, T S, size_t image_size); +/* + * In place add scalar S to scalar array A + * + * A[i] = A[i]*S + */ +template +void cpu_kernel_add( + T *A, + T S, + size_t size +); /* void finalizeMstddev( int blockIdx_x, int threadIdx_x, diff --git a/src/acc/cuda/cuda_kernels/helper.cuh b/src/acc/cuda/cuda_kernels/helper.cuh index b4b8579c8..8019acf4e 100644 --- a/src/acc/cuda/cuda_kernels/helper.cuh +++ b/src/acc/cuda/cuda_kernels/helper.cuh @@ -466,6 +466,21 @@ __global__ void cuda_kernel_multi( A[pixel] = A[pixel]*S; } +/* + * In place add scalar S to scalar array A + * + * A[i] = A[i]*S + */ +template +__global__ void cuda_kernel_add( + T *A, + T S, + int size) +{ + int pixel = threadIdx.x + blockIdx.x * blockDim.x; + if (pixel < size) + A[pixel] += S; +} } /* diff --git a/src/acc/utilities.h b/src/acc/utilities.h index e3d4ebffd..6bb56d2fc 100644 --- a/src/acc/utilities.h +++ b/src/acc/utilities.h @@ -63,6 +63,25 @@ CudaKernels::cuda_kernel_multi<<>>( #endif } +template +static void add(int block_size, cudaStream_t stream, T *array, T value, size_t size) +{ +#ifdef _CUDA_ENABLED + size_t MultiBsize = ( (size_t) ceilf((float)size/(float)BLOCK_SIZE)); + CudaKernels::cuda_kernel_add<<>>( + array, + value, + size + ); +#else + CpuKernels::cpu_kernel_add( + array, + value, + size + ); +#endif +} + template static void translate(int block_size, AccDataTypes::Image &in, diff --git a/src/align_tiltseries_runner.cpp b/src/align_tiltseries_runner.cpp index 24a1244ef..87836ce98 100644 --- a/src/align_tiltseries_runner.cpp +++ b/src/align_tiltseries_runner.cpp @@ -291,7 +291,7 @@ void AlignTiltseriesRunner::joinImodWrapperResults() } else { - MDout.setValue(EMDL_TOMO_ETOMO_DIRECTIVE_FILE, "undefined", itomo); + MDout.setValue(EMDL_TOMO_ETOMO_DIRECTIVE_FILE, std::string("undefined"), itomo); } } else if (verb) From 535dcae2744a529cb158ee658d85182726bc3f8d Mon Sep 17 00:00:00 2001 From: scheres Date: Fri, 8 Jul 2022 09:37:29 +0100 Subject: [PATCH 080/495] deal with dimensionality of the shifts for mydata.is_tomo --- src/acc/acc_ml_optimiser_impl.h | 89 ++++++++++++--------- src/acc/cpu/cpu_ml_optimiser.h | 2 + src/acc/cuda/cuda_ml_optimiser.h | 2 + src/ml_optimiser.cpp | 132 ++++++++++++++++--------------- 4 files changed, 123 insertions(+), 102 deletions(-) diff --git a/src/acc/acc_ml_optimiser_impl.h b/src/acc/acc_ml_optimiser_impl.h index c0f62ac2c..588aa4d29 100644 --- a/src/acc/acc_ml_optimiser_impl.h +++ b/src/acc/acc_ml_optimiser_impl.h @@ -23,7 +23,8 @@ void getFourierTransformsAndCtfs(long int part_id, CUSTOM_ALLOCATOR_REGION_NAME("GFTCTF"); Matrix2D Aori; - Matrix1D my_projected_com(baseMLO->mymodel.data_dim), my_refined_ibody_offset(baseMLO->mymodel.data_dim); + int shiftdim = (accMLO->shiftsIs3D) ? 3 : 2 ; + Matrix1D my_projected_com(shiftdim), my_refined_ibody_offset(shiftdim); // Get the norm_correction RFLOAT normcorr = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_NORM); @@ -42,7 +43,7 @@ void getFourierTransformsAndCtfs(long int part_id, // Get the optimal origin offsets from the previous iteration // Sjors 5mar18: it is very important that my_old_offset has baseMLO->mymodel.data_dim and not just (3), as transformCartesianAndHelicalCoords will give different results!!! - Matrix1D my_old_offset(baseMLO->mymodel.data_dim), my_prior(baseMLO->mymodel.data_dim), my_old_offset_ori; + Matrix1D my_old_offset(shiftdim), my_prior(shiftdim), my_old_offset_ori; int icol_rot, icol_tilt, icol_psi, icol_xoff, icol_yoff, icol_zoff; XX(my_old_offset) = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_XOFF); YY(my_old_offset) = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_YOFF); @@ -54,7 +55,7 @@ void getFourierTransformsAndCtfs(long int part_id, if (YY(my_prior) > 998.99 && YY(my_prior) < 999.01) YY(my_prior) = 0.; - if (accMLO->dataIs3D) + if (accMLO->shiftsIs3D) { ZZ(my_old_offset) = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_ZOFF); ZZ(my_prior) = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_ZOFF_PRIOR); @@ -72,8 +73,9 @@ void getFourierTransformsAndCtfs(long int part_id, DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_TILT), DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_PSI), Aori, false); my_projected_com = Aori * baseMLO->mymodel.com_bodies[ibody]; - // This will have made my_projected_com of size 3 again! resize to mymodel.data_dim - my_projected_com.resize(baseMLO->mymodel.data_dim); + // This will have made my_projected_com of size 3 again! resize to shiftdim + int shiftdim = (accMLO->shiftsIs3D) ? 3 : 2 ; + my_projected_com.resize(shiftdim); // Subtract the projected COM offset, to position this body in the center // Also keep the my_old_offset in my_old_offset_ori @@ -86,7 +88,7 @@ void getFourierTransformsAndCtfs(long int part_id, icol_zoff = 5 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; XX(my_refined_ibody_offset) = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, icol_xoff); YY(my_refined_ibody_offset) = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, icol_yoff); - if (baseMLO->mymodel.data_dim == 3) + if (accMLO->shiftsIs3D) ZZ(my_refined_ibody_offset) = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, icol_zoff); // For multi-body refinement: set the priors of the translations to zero (i.e. everything centred around consensus offset) @@ -196,10 +198,10 @@ void getFourierTransformsAndCtfs(long int part_id, bool do_local_angular_searches = (do_auto_refine_local_searches) || (do_classification_local_searches); if (!do_local_angular_searches) { - if (! accMLO->dataIs3D) - XX(my_old_offset_helix_coords) = 0.; - else + if (accMLO->shiftsIs3D) ZZ(my_old_offset_helix_coords) = 0.; + else + XX(my_old_offset_helix_coords) = 0.; } } // TODO: Now re-calculate the my_old_offset in the real (or image) system of coordinate (rotate -psi angle) @@ -438,7 +440,7 @@ void getFourierTransformsAndCtfs(long int part_id, normcorr_val, XX(my_old_offset), YY(my_old_offset), - (accMLO->dataIs3D) ? ZZ(my_old_offset) : 0., + (accMLO->shiftsIs3D) ? ZZ(my_old_offset) : 0., accMLO->dataIs3D); LAUNCH_PRIVATE_ERROR(cudaGetLastError(),accMLO->errorStatus); @@ -458,7 +460,7 @@ void getFourierTransformsAndCtfs(long int part_id, normcorr_val, XX(my_old_offset), YY(my_old_offset), - (accMLO->dataIs3D) ? ZZ(my_old_offset) : 0., + (accMLO->shiftsIs3D) ? ZZ(my_old_offset) : 0., accMLO->dataIs3D); LAUNCH_PRIVATE_ERROR(cudaGetLastError(),accMLO->errorStatus); CTOC(accMLO->timer,"TranslateAndNormCorrect_recImg"); @@ -798,6 +800,7 @@ void getFourierTransformsAndCtfs(long int part_id, Abody = Aori * (baseMLO->mymodel.orient_bodies[obody]).transpose() * baseMLO->A_rot90 * Aresi * baseMLO->mymodel.orient_bodies[obody]; // Apply anisotropic mag and scaling + if (baseMLO->mydata.is_tomo) Abody = baseMLO->mydata.getRotationMatrix(part_id, img_id) * Abody; Abody = baseMLO->mydata.obsModel.applyAnisoMag(Abody, optics_group); Abody = baseMLO->mydata.obsModel.applyScaleDifference(Abody, optics_group, baseMLO->mymodel.ori_size, baseMLO->mymodel.pixel_size); @@ -818,12 +821,13 @@ void getFourierTransformsAndCtfs(long int part_id, // 17May2017: Body is centered at its own COM // move it back to its place in the original particle image - Matrix1D other_projected_com(baseMLO->mymodel.data_dim); + int shiftdim = (accMLO->shiftsIs3D) ? 3 : 2 ; + Matrix1D other_projected_com(shiftdim); // Projected COM for this body (using Aori, just like above for ibody and my_projected_com!!!) other_projected_com = Aori * (baseMLO->mymodel.com_bodies[obody]); - // This will have made other_projected_com of size 3 again! resize to mymodel.data_dim - other_projected_com.resize(baseMLO->mymodel.data_dim); + // This will have made other_projected_com of size 3 again! resize to shiftdim + other_projected_com.resize(shiftdim); // Do the exact same as was done for the ibody, but DONT selfROUND here, as later phaseShift applied to ibody below!!! other_projected_com -= my_old_offset_ori; @@ -831,14 +835,15 @@ void getFourierTransformsAndCtfs(long int part_id, // Subtract refined obody-displacement XX(other_projected_com) -= DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, ocol_xoff); YY(other_projected_com) -= DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, ocol_yoff); - if (baseMLO->mymodel.data_dim == 3) + if (accMLO->shiftsIs3D) ZZ(other_projected_com) -= DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, ocol_zoff); // Add the my_old_offset=selfRound(my_old_offset_ori - my_projected_com) already applied to this image for ibody other_projected_com += my_old_offset; shiftImageInFourierTransform(FTo, Faux, (RFLOAT)baseMLO->image_full_size[optics_group], - XX(other_projected_com), YY(other_projected_com), (accMLO->dataIs3D) ? ZZ(other_projected_com) : 0.); + XX(other_projected_com), YY(other_projected_com), + (accMLO->shiftsIs3D) ? ZZ(other_projected_com) : 0.); // Sum the Fourier transforms of all the obodies Fsum_obody += Faux; @@ -885,10 +890,12 @@ void getFourierTransformsAndCtfs(long int part_id, // 23jul17: NEW: as we haven't applied the (nonROUNDED!!) my_refined_ibody_offset yet, do this now in the FourierTransform Faux = op.Fimg.at(img_id); shiftImageInFourierTransform(Faux, op.Fimg.at(img_id), (RFLOAT)baseMLO->image_full_size[optics_group], - XX(my_refined_ibody_offset), YY(my_refined_ibody_offset), (accMLO->dataIs3D) ? ZZ(my_refined_ibody_offset) : 0); + XX(my_refined_ibody_offset), YY(my_refined_ibody_offset), + (accMLO->shiftsIs3D) ? ZZ(my_refined_ibody_offset) : 0); Faux = op.Fimg_nomask.at(img_id); shiftImageInFourierTransform(Faux, op.Fimg_nomask.at(img_id), (RFLOAT)baseMLO->image_full_size[optics_group], - XX(my_refined_ibody_offset), YY(my_refined_ibody_offset), (accMLO->dataIs3D) ? ZZ(my_refined_ibody_offset) : 0); + XX(my_refined_ibody_offset), YY(my_refined_ibody_offset), + (accMLO->shiftsIs3D) ? ZZ(my_refined_ibody_offset) : 0); } // end if mymodel.nr_bodies > 1 @@ -1061,7 +1068,7 @@ void getAllSquaredDifferencesCoarse( xshift = oversampled_translations_x[0]; yshift = oversampled_translations_y[0]; - if (accMLO->dataIs3D) + if (accMLO->shiftsIs3D) zshift = oversampled_translations_z[0]; if ( (baseMLO->do_helical_refine) && (! baseMLO->ignore_helical_symmetry) ) @@ -1069,7 +1076,8 @@ void getAllSquaredDifferencesCoarse( RFLOAT rot_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_ROT); RFLOAT tilt_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_TILT); RFLOAT psi_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata,op.metadata_offset, METADATA_PSI); - transformCartesianAndHelicalCoords(xshift, yshift, zshift, xshift, yshift, zshift, rot_deg, tilt_deg, psi_deg, (accMLO->dataIs3D) ? (3) : (2), HELICAL_TO_CART_COORDS); + transformCartesianAndHelicalCoords(xshift, yshift, zshift, xshift, yshift, zshift, rot_deg, tilt_deg, psi_deg, + (accMLO->shiftsIs3D) ? (3) : (2), HELICAL_TO_CART_COORDS); } trans_xyz[trans_x_offset+itrans] = -2 * PI * xshift / (double)baseMLO->image_full_size[optics_group]; @@ -1285,7 +1293,7 @@ void getAllSquaredDifferencesFine( xshift = oversampled_translations_x[iover_trans]; yshift = oversampled_translations_y[iover_trans]; - if (accMLO->dataIs3D) + if (accMLO->shiftsIs3D) zshift = oversampled_translations_z[iover_trans]; if ( (baseMLO->do_helical_refine) && (! baseMLO->ignore_helical_symmetry) ) @@ -1294,7 +1302,8 @@ void getAllSquaredDifferencesFine( RFLOAT rot_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_ROT); RFLOAT tilt_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_TILT); RFLOAT psi_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_PSI); - transformCartesianAndHelicalCoords(xshift, yshift, zshift, xshift, yshift, zshift, rot_deg, tilt_deg, psi_deg, (accMLO->dataIs3D) ? (3) : (2), HELICAL_TO_CART_COORDS); + transformCartesianAndHelicalCoords(xshift, yshift, zshift, xshift, yshift, zshift, rot_deg, tilt_deg, psi_deg, + (accMLO->shiftsIs3D) ? (3) : (2), HELICAL_TO_CART_COORDS); } trans_xyz[trans_x_offset+j] = -2 * PI * xshift / (double)baseMLO->image_full_size[optics_group]; @@ -1651,7 +1660,7 @@ void convertAllSquaredDifferencesToWeights(unsigned exp_ipass, { old_offset_x = XX(op.old_offset); old_offset_y = YY(op.old_offset); - if (accMLO->dataIs3D) + if (accMLO->shiftsIs3D) old_offset_z = ZZ(op.old_offset); } @@ -1727,7 +1736,7 @@ void convertAllSquaredDifferencesToWeights(unsigned exp_ipass, { myprior_x = XX(op.prior); myprior_y = YY(op.prior); - if (accMLO->dataIs3D) + if (accMLO->shiftsIs3D) myprior_z = ZZ(op.prior); } @@ -1738,7 +1747,7 @@ void convertAllSquaredDifferencesToWeights(unsigned exp_ipass, if ( (baseMLO->do_helical_refine) && (! baseMLO->ignore_helical_symmetry)) { RFLOAT mypriors_len2 = myprior_x * myprior_x + myprior_y * myprior_y; - if (accMLO->dataIs3D) + if (accMLO->shiftsIs3D) mypriors_len2 += myprior_z * myprior_z; if (mypriors_len2 > 0.00001) @@ -1746,7 +1755,8 @@ void convertAllSquaredDifferencesToWeights(unsigned exp_ipass, RFLOAT rot_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_ROT); RFLOAT tilt_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_TILT); RFLOAT psi_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_PSI); - transformCartesianAndHelicalCoords(myprior_x, myprior_y, myprior_z, myprior_x, myprior_y, myprior_z, rot_deg, tilt_deg, psi_deg, (accMLO->dataIs3D) ? (3) : (2), CART_TO_HELICAL_COORDS); + transformCartesianAndHelicalCoords(myprior_x, myprior_y, myprior_z, myprior_x, myprior_y, myprior_z, rot_deg, tilt_deg, psi_deg, + (accMLO->shiftsIs3D) ? (3) : (2), CART_TO_HELICAL_COORDS); } } // (For helical refinement) Now offset, old_offset, sampling.translations and myprior are all in helical coordinates @@ -1758,10 +1768,10 @@ void convertAllSquaredDifferencesToWeights(unsigned exp_ipass, RFLOAT offset_y = old_offset_y + baseMLO->sampling.translations_y[itrans]; double tdiff2 = 0.; - if ( (! baseMLO->do_helical_refine) || (baseMLO->ignore_helical_symmetry) || (accMLO->dataIs3D) ) + if ( (! baseMLO->do_helical_refine) || (baseMLO->ignore_helical_symmetry) || (accMLO->shiftsIs3D) ) tdiff2 += (offset_x - myprior_x) * (offset_x - myprior_x); tdiff2 += (offset_y - myprior_y) * (offset_y - myprior_y); - if (accMLO->dataIs3D) + if (accMLO->shiftsIs3D) { RFLOAT offset_z = old_offset_z + baseMLO->sampling.translations_z[itrans]; if ( (! baseMLO->do_helical_refine) || (baseMLO->ignore_helical_symmetry) ) @@ -2241,7 +2251,7 @@ void storeWeightedSums(OptimisationParamters &op, SamplingParameters &sp, { myprior_x = XX(op.prior); myprior_y = YY(op.prior); - if (baseMLO->mymodel.data_dim == 3) + if (accMLO->shiftsIs3D) { myprior_z = ZZ(op.prior); old_offset_z = ZZ(op.old_offset); @@ -2262,12 +2272,12 @@ void storeWeightedSums(OptimisationParamters &op, SamplingParameters &sp, { oo_otrans[otrans_x+fake_class*nr_transes+iitrans] = old_offset_x + oversampled_translations_x[iover_trans]; oo_otrans[otrans_y+fake_class*nr_transes+iitrans] = old_offset_y + oversampled_translations_y[iover_trans]; - if (accMLO->dataIs3D) + if (accMLO->shiftsIs3D) oo_otrans[otrans_z+fake_class*nr_transes+iitrans] = old_offset_z + oversampled_translations_z[iover_trans]; // Calculate the vector length of myprior RFLOAT mypriors_len2 = myprior_x * myprior_x + myprior_y * myprior_y; - if (accMLO->dataIs3D) + if (accMLO->shiftsIs3D) mypriors_len2 += myprior_z * myprior_z; // If it is doing helical refinement AND Cartesian vector myprior has a length > 0, transform the vector to its helical coordinates @@ -2276,7 +2286,8 @@ void storeWeightedSums(OptimisationParamters &op, SamplingParameters &sp, RFLOAT rot_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_ROT); RFLOAT tilt_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_TILT); RFLOAT psi_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_PSI); - transformCartesianAndHelicalCoords(myprior_x, myprior_y, myprior_z, myprior_x, myprior_y, myprior_z, rot_deg, tilt_deg, psi_deg, (accMLO->dataIs3D) ? (3) : (2), CART_TO_HELICAL_COORDS); + transformCartesianAndHelicalCoords(myprior_x, myprior_y, myprior_z, myprior_x, myprior_y, myprior_z, rot_deg, tilt_deg, psi_deg, + (accMLO->shiftsIs3D) ? (3) : (2), CART_TO_HELICAL_COORDS); } if ( (! baseMLO->do_helical_refine) || (baseMLO->ignore_helical_symmetry) ) @@ -2284,7 +2295,7 @@ void storeWeightedSums(OptimisationParamters &op, SamplingParameters &sp, RFLOAT diffx = myprior_x - oo_otrans[otrans_x+fake_class*nr_transes+iitrans]; RFLOAT diffy = myprior_y - oo_otrans[otrans_y+fake_class*nr_transes+iitrans]; RFLOAT diffz = 0; - if (accMLO->dataIs3D) + if (accMLO->shiftsIs3D) diffz = myprior_z - (old_offset_z + oversampled_translations_z[iover_trans]); oo_otrans[otrans_x2y2z2+fake_class*nr_transes+iitrans] = diffx*diffx + diffy*diffy + diffz*diffz; @@ -2442,11 +2453,12 @@ void storeWeightedSums(OptimisationParamters &op, SamplingParameters &sp, RFLOAT old_psi = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, icol_psi); DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, icol_psi) = psi; - Matrix1D shifts(baseMLO->mymodel.data_dim); + int shiftdim = (accMLO->shiftsIs3D) ? 3 : 2 ; + Matrix1D shifts(shiftdim); XX(shifts) = XX(op.old_offset) + oversampled_translations_x[op.max_index.iovertrans]; YY(shifts) = YY(op.old_offset) + oversampled_translations_y[op.max_index.iovertrans]; - if (accMLO->dataIs3D) + if (accMLO->shiftsIs3D) { ZZ(shifts) = ZZ(op.old_offset) + oversampled_translations_z[op.max_index.iovertrans]; } @@ -2457,7 +2469,7 @@ void storeWeightedSums(OptimisationParamters &op, SamplingParameters &sp, DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, icol_xoff) = XX(shifts); DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, icol_yoff) = YY(shifts); - if (accMLO->dataIs3D) + if (accMLO->shiftsIs3D) DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, icol_zoff) = ZZ(shifts); if (ibody == 0) @@ -2515,7 +2527,7 @@ void storeWeightedSums(OptimisationParamters &op, SamplingParameters &sp, xshift = oversampled_translations_x[iover_trans]; yshift = oversampled_translations_y[iover_trans]; - if (accMLO->dataIs3D) + if (accMLO->shiftsIs3D) zshift = oversampled_translations_z[iover_trans]; if ( (baseMLO->do_helical_refine) && (! baseMLO->ignore_helical_symmetry) ) @@ -2523,7 +2535,8 @@ void storeWeightedSums(OptimisationParamters &op, SamplingParameters &sp, RFLOAT rot_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_ROT); RFLOAT tilt_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_TILT); RFLOAT psi_deg = DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_PSI); - transformCartesianAndHelicalCoords(xshift, yshift, zshift, xshift, yshift, zshift, rot_deg, tilt_deg, psi_deg, (accMLO->dataIs3D) ? (3) : (2), HELICAL_TO_CART_COORDS); + transformCartesianAndHelicalCoords(xshift, yshift, zshift, xshift, yshift, zshift, rot_deg, tilt_deg, psi_deg, + (accMLO->shiftsIs3D) ? (3) : (2), HELICAL_TO_CART_COORDS); } trans_xyz[trans_x_offset+j] = -2 * PI * xshift / (double)baseMLO->image_full_size[optics_group]; diff --git a/src/acc/cpu/cpu_ml_optimiser.h b/src/acc/cpu/cpu_ml_optimiser.h index 132f97784..cc26969a9 100644 --- a/src/acc/cpu/cpu_ml_optimiser.h +++ b/src/acc/cpu/cpu_ml_optimiser.h @@ -49,6 +49,7 @@ class MlOptimiserCpu bool refIs3D; bool dataIs3D; + bool shiftsIs3D; int thread_id; @@ -69,6 +70,7 @@ class MlOptimiserCpu transformer2(baseMLOptimiser->mymodel.data_dim), refIs3D(baseMLO->mymodel.ref_dim == 3), dataIs3D(baseMLO->mymodel.data_dim == 3), + shiftsIs3D(baseMLO->mymodel.data_dim == 3 || baseMLO->mydata.is_tomo), #ifdef TIMING_FILES timer(timing_fnm), #endif diff --git a/src/acc/cuda/cuda_ml_optimiser.h b/src/acc/cuda/cuda_ml_optimiser.h index e9f91329b..2a44a2f8c 100644 --- a/src/acc/cuda/cuda_ml_optimiser.h +++ b/src/acc/cuda/cuda_ml_optimiser.h @@ -91,6 +91,7 @@ class MlOptimiserCuda bool refIs3D; bool dataIs3D; + bool shiftsIs3D; int device_id; @@ -113,6 +114,7 @@ class MlOptimiserCuda transformer2(cudaStreamPerThread, bundle->allocator, baseMLOptimiser->mymodel.data_dim), refIs3D(baseMLO->mymodel.ref_dim == 3), dataIs3D(baseMLO->mymodel.data_dim == 3), + shiftsIs3D(baseMLO->mymodel.data_dim == 3 || baseMLO->mydata.is_tomo), bundle(bundle), device_id(bundle->device_id), #ifdef TIMING_FILES diff --git a/src/ml_optimiser.cpp b/src/ml_optimiser.cpp index 730d06df2..214cfe050 100644 --- a/src/ml_optimiser.cpp +++ b/src/ml_optimiser.cpp @@ -2050,7 +2050,7 @@ void MlOptimiser::initialiseGeneral(int rank) sampling.offset_range *= mymodel.pixel_size; sampling.offset_step *= mymodel.pixel_size; } - sampling.initialise(mymodel.ref_dim, (mymodel.data_dim == 3), do_gpu, (verb>0), + sampling.initialise(mymodel.ref_dim, (mymodel.data_dim == 3 || mydata.is_tomo), do_gpu, (verb>0), do_local_searches_helical, (do_helical_refine) && (!ignore_helical_symmetry), helical_rise_initial, helical_twist_initial); @@ -2132,7 +2132,7 @@ void MlOptimiser::initialiseGeneral(int rank) sum_changes_optimal_classes = 0.; sum_changes_count = 0.; - if (mymodel.data_dim == 3) + if (mymodel.data_dim == 3 || mydata.is_tomo) { // TODO: later do norm correction?! @@ -2144,7 +2144,8 @@ void MlOptimiser::initialiseGeneral(int rank) if (do_skip_align) do_shifts_onthefly = false; // on-the-fly shifts are incompatible with do_skip_align! - // getMetaAndImageData is not made for passing multiple volumes! + + // getMetaAndImageData is not made for passing multiple volumes! do_parallel_disc_io = true; } @@ -3104,7 +3105,7 @@ void MlOptimiser::iterate() mymodel.helical_rise, mymodel.helical_twist, helical_nstart, - (mymodel.data_dim == 3), + (mymodel.data_dim == 3 || mydata.is_tomo), (do_auto_refine || do_auto_sampling), mymodel.sigma2_rot, mymodel.sigma2_tilt, @@ -3877,7 +3878,7 @@ void MlOptimiser::expectationSomeParticles(long int my_first_part_id, long int m my_old_offset_y = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_YOFF); rounded_offset_x = my_old_offset_x - ROUND(my_old_offset_x); rounded_offset_y = my_old_offset_y - ROUND(my_old_offset_y); - if (mymodel.data_dim == 3) + if (mymodel.data_dim == 3 || mydata.is_tomo) { my_old_offset_z = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ZOFF); rounded_offset_z = my_old_offset_z - ROUND(my_old_offset_z); @@ -4867,7 +4868,7 @@ void MlOptimiser::maximizationOtherParameters() if (!fix_sigma_offset && mymodel.nr_bodies == 1) { mymodel.sigma2_offset *= my_mu; - if (mymodel.data_dim == 3) + if (mymodel.data_dim == 3 || mydata.is_tomo) { if ( (do_helical_refine) && (!ignore_helical_symmetry) ) mymodel.sigma2_offset += (1. - my_mu) * (wsum_model.sigma2_offset) / (2. * sum_weight); @@ -5497,7 +5498,8 @@ void MlOptimiser::getFourierTransformsAndCtfs( { Matrix2D Aori; - Matrix1D my_projected_com(mymodel.data_dim), my_refined_ibody_offset(mymodel.data_dim); + int shiftdim = (mymodel.data_dim == 3 || mydata.is_tomo) ? 3 : 2 ; + Matrix1D my_projected_com(shiftdim), my_refined_ibody_offset(shiftdim); int exp_nr_images = mydata.numberOfImagesInParticle(part_id); // Get the norm_correction (for multi-body refinement: still use the one from the consensus refinement!) @@ -5516,14 +5518,14 @@ void MlOptimiser::getFourierTransformsAndCtfs( // Get the old offsets and the priors on the offsets // Sjors 5mar18: it is very important that my_old_offset has baseMLO->mymodel.data_dim and not just (3), as transformCartesianAndHelicalCoords will give different results!!! - Matrix1D my_old_offset(mymodel.data_dim), my_prior(mymodel.data_dim), my_old_offset_ori; + Matrix1D my_old_offset(shiftdim), my_prior(shiftdim), my_old_offset_ori; int icol_rot, icol_tilt, icol_psi, icol_xoff, icol_yoff, icol_zoff; XX(my_old_offset) = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_XOFF); XX(my_prior) = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_XOFF_PRIOR); YY(my_old_offset) = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_YOFF); YY(my_prior) = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_YOFF_PRIOR); - if (mymodel.data_dim == 3) + if (mymodel.data_dim == 3 || mydata.is_tomo) { ZZ(my_old_offset) = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ZOFF); ZZ(my_prior) = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ZOFF_PRIOR); @@ -5538,7 +5540,8 @@ void MlOptimiser::getFourierTransformsAndCtfs( DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PSI), Aori, false); my_projected_com = Aori * mymodel.com_bodies[ibody]; // This will have made my_projected_com of size 3 again! resize to mymodel.data_dim - my_projected_com.resize(mymodel.data_dim); + int shiftdim = (mymodel.data_dim == 3 || mydata.is_tomo) ? 3 : 2 ; + my_projected_com.resize(shiftdim); #ifdef DEBUG_BODIES @@ -5560,7 +5563,7 @@ void MlOptimiser::getFourierTransformsAndCtfs( icol_zoff = 5 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS; XX(my_refined_ibody_offset) = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_xoff); YY(my_refined_ibody_offset) = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_yoff); - if (mymodel.data_dim == 3) + if (mymodel.data_dim == 3 || mydata.is_tomo) ZZ(my_refined_ibody_offset) = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_zoff); // For multi-body refinement: set the priors of the translations to zero (i.e. everything centred around consensus offset) @@ -5583,7 +5586,7 @@ void MlOptimiser::getFourierTransformsAndCtfs( XX(my_prior) = 0.; if (YY(my_prior) > 998.99 && YY(my_prior) < 999.01) YY(my_prior) = 0.; - if (mymodel.data_dim == 3 && ZZ(my_prior) > 998.99 && ZZ(my_prior) < 999.01) + if (mymodel.data_dim == 3 || mydata.is_tomo && ZZ(my_prior) > 998.99 && ZZ(my_prior) < 999.01) ZZ(my_prior) = 0.; // Store the prior on translations @@ -5786,7 +5789,6 @@ void MlOptimiser::getFourierTransformsAndCtfs( RFLOAT my_pixel_size = mydata.getOpticsPixelSize(optics_group); int my_image_size = mydata.getOpticsImageSize(optics_group); - // Get the image and recimg data if (do_parallel_disc_io) { @@ -6264,12 +6266,14 @@ void MlOptimiser::getFourierTransformsAndCtfs( // 17May2017: Body is centered at its own COM // move it back to its place in the original particle image - Matrix1D other_projected_com(mymodel.data_dim); + int shiftdim = (mymodel.data_dim == 3 || mydata.is_tomo) ? 3 : 2 ; + Matrix1D other_projected_com(shiftdim); // Projected COM for this body (using Aori, just like above for ibody and my_projected_com!!!) other_projected_com = Aori * (mymodel.com_bodies[obody]); - // This will have made other_projected_com of size 3 again! resize to mymodel.data_dim - other_projected_com.resize(mymodel.data_dim); + + // This will have made other_projected_com of size 3 again! resize to mymodel.data_dim + other_projected_com.resize(shiftdim); // Do the exact same as was done for the ibody, but DONT selfROUND here, as later phaseShift applied to ibody below!!! other_projected_com -= my_old_offset_ori; @@ -6284,7 +6288,7 @@ void MlOptimiser::getFourierTransformsAndCtfs( // Subtract refined obody-displacement XX(other_projected_com) -= DIRECT_A2D_ELEM(exp_metadata, metadata_offset, ocol_xoff); YY(other_projected_com) -= DIRECT_A2D_ELEM(exp_metadata, metadata_offset, ocol_yoff); - if (mymodel.data_dim == 3) + if (mymodel.data_dim == 3 || mydata.is_tomo) { ZZ(other_projected_com) -= DIRECT_A2D_ELEM(exp_metadata, metadata_offset, ocol_zoff); } @@ -6615,7 +6619,7 @@ void MlOptimiser::precalculateShiftedImagesCtfsAndInvSigma2s(bool do_also_unmask xshift = oversampled_translations_x[iover_trans]; yshift = oversampled_translations_y[iover_trans]; - if (mymodel.data_dim == 3) + if (mymodel.data_dim == 3 || mydata.is_tomo) zshift = oversampled_translations_z[iover_trans]; if ( (do_helical_refine) && (!ignore_helical_symmetry) ) @@ -6632,6 +6636,8 @@ void MlOptimiser::precalculateShiftedImagesCtfsAndInvSigma2s(bool do_also_unmask #endif } + //// XXXXXXXXXX make 3D shift into 2D shift + if (mydata.is_tomo) REPORT_ERROR("TODO: think about 2D shifts here!!"); // Shift through phase-shifts in the Fourier transform @@ -6805,7 +6811,6 @@ void MlOptimiser::getAllSquaredDifferences(long int part_id, int ibody, if (mymodel.nr_bodies > 1) { - // ipart=0 because in multi-body refinement we do not do movie frames! RFLOAT rot_ori = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ROT); RFLOAT tilt_ori = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_TILT); RFLOAT psi_ori = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PSI); @@ -6878,13 +6883,6 @@ void MlOptimiser::getAllSquaredDifferences(long int part_id, int ibody, oversampled_tilt[iover_rot], oversampled_psi[iover_rot], A, false); - Matrix2D Aproj; - if (mydata.is_tomo) - { - Aproj = mydata.getRotationMatrix(part_id, img_id); - REPORT_ERROR("ERROR: TODO: implement rotations of img_id tilt series"); - - } // Project the reference map (into Fref) #ifdef TIMING // Only time one thread, as I also only time one MPI process @@ -6897,13 +6895,15 @@ void MlOptimiser::getAllSquaredDifferences(long int part_id, int ibody, if (mymodel.nr_bodies > 1) { Abody = Aori * (mymodel.orient_bodies[ibody]).transpose() * A_rot90 * A * mymodel.orient_bodies[ibody]; - Abody = mydata.obsModel.applyAnisoMag(Abody, optics_group); + if (mydata.is_tomo) Abody = mydata.getRotationMatrix(part_id, img_id) * Abody; + Abody = mydata.obsModel.applyAnisoMag(Abody, optics_group); Abody = mydata.obsModel.applyScaleDifference(Abody, optics_group, mymodel.ori_size, mymodel.pixel_size); - (mymodel.PPref[ibody]).get2DFourierTransform(Fref, Abody); + (mymodel.PPref[ibody]).get2DFourierTransform(Fref, Abody); } else { - A = mydata.obsModel.applyAnisoMag(A, optics_group); + if (mydata.is_tomo) A = mydata.getRotationMatrix(part_id, img_id) * A; + A = mydata.obsModel.applyAnisoMag(A, optics_group); A = mydata.obsModel.applyScaleDifference(A, optics_group, mymodel.ori_size, mymodel.pixel_size); (mymodel.PPref[exp_iclass]).get2DFourierTransform(Fref, A); } @@ -6995,12 +6995,13 @@ void MlOptimiser::getAllSquaredDifferences(long int part_id, int ibody, if ( (do_helical_refine) && (!ignore_helical_symmetry) ) { bool use_coarse_size = false; + RFLOAT xshift_in = 0., yshift_in = 0., zshift_in = 0.; RFLOAT xshift = 0., yshift = 0., zshift = 0.; - xshift = (exp_current_oversampling == 0) ? (oversampled_translations_x[0]) : (oversampled_translations_x[iover_trans]); - yshift = (exp_current_oversampling == 0) ? (oversampled_translations_y[0]) : (oversampled_translations_y[iover_trans]); - if (mymodel.data_dim == 3) - zshift = (exp_current_oversampling == 0) ? (oversampled_translations_z[0]) : (oversampled_translations_z[iover_trans]); + xshift_in = (exp_current_oversampling == 0) ? (oversampled_translations_x[0]) : (oversampled_translations_x[iover_trans]); + yshift_in = (exp_current_oversampling == 0) ? (oversampled_translations_y[0]) : (oversampled_translations_y[iover_trans]); + if (mymodel.data_dim == 3 || mydata.is_tomo) + zshift_in = (exp_current_oversampling == 0) ? (oversampled_translations_z[0]) : (oversampled_translations_z[iover_trans]); if (mydata.is_tomo) REPORT_ERROR("ERROR; TODO: think about the below for 2D shifts in stacks..."); @@ -7008,7 +7009,7 @@ void MlOptimiser::getAllSquaredDifferences(long int part_id, int ibody, RFLOAT tilt_deg = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_TILT); RFLOAT psi_deg = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PSI); transformCartesianAndHelicalCoords( - xshift, yshift, zshift, + xshift_in, yshift_in, zshift_in, xshift, yshift, zshift, rot_deg, tilt_deg, psi_deg, mymodel.data_dim, @@ -7391,7 +7392,7 @@ void MlOptimiser::convertAllSquaredDifferencesToWeights(long int part_id, int ib { old_offset_x = XX(exp_old_offset); old_offset_y = YY(exp_old_offset); - if (mymodel.data_dim == 3) + if (mymodel.data_dim == 3 || mydata.is_tomo) old_offset_z = ZZ(exp_old_offset); } @@ -7458,7 +7459,7 @@ void MlOptimiser::convertAllSquaredDifferencesToWeights(long int part_id, int ib { myprior_x = XX(exp_prior); myprior_y = YY(exp_prior); - if (mymodel.data_dim == 3) + if (mymodel.data_dim == 3 || mydata.is_tomo) myprior_z = ZZ(exp_prior); } for (long int itrans = exp_itrans_min; itrans <= exp_itrans_max; itrans++) @@ -7466,10 +7467,10 @@ void MlOptimiser::convertAllSquaredDifferencesToWeights(long int part_id, int ib RFLOAT offset_x = old_offset_x + sampling.translations_x[itrans]; RFLOAT offset_y = old_offset_y + sampling.translations_y[itrans]; RFLOAT tdiff2 = 0.; - if ( (!do_helical_refine) || (ignore_helical_symmetry) || (mymodel.data_dim == 3) ) + if ( (!do_helical_refine) || (ignore_helical_symmetry) || (mymodel.data_dim == 3 || mydata.is_tomo) ) tdiff2 += (offset_x - myprior_x) * (offset_x - myprior_x); tdiff2 += (offset_y - myprior_y) * (offset_y - myprior_y); - if (mymodel.data_dim == 3) + if (mymodel.data_dim == 3 || mydata.is_tomo) { RFLOAT offset_z = old_offset_z + sampling.translations_z[itrans]; if ( (!do_helical_refine) || (ignore_helical_symmetry) ) @@ -7510,7 +7511,7 @@ void MlOptimiser::convertAllSquaredDifferencesToWeights(long int part_id, int ib { myprior_x = XX(exp_prior); myprior_y = YY(exp_prior); - if (mymodel.data_dim == 3) + if (mymodel.data_dim == 3 || mydata.is_tomo) myprior_z = ZZ(exp_prior); } @@ -7549,7 +7550,7 @@ void MlOptimiser::convertAllSquaredDifferencesToWeights(long int part_id, int ib // May18,2015 - Shaoda & Sjors - Helical refinement (translational searches) // Calculate the vector length of myprior RFLOAT mypriors_len2 = myprior_x * myprior_x + myprior_y * myprior_y; - if (mymodel.data_dim == 3) + if (mymodel.data_dim == 3 || mydata.is_tomo) mypriors_len2 += myprior_z * myprior_z; // If it is doing helical refinement AND Cartesian vector myprior has a length > 0, transform the vector to its helical coordinates if ( (do_helical_refine) && (!ignore_helical_symmetry) && (mypriors_len2 > 0.00001) ) @@ -7566,10 +7567,10 @@ void MlOptimiser::convertAllSquaredDifferencesToWeights(long int part_id, int ib RFLOAT offset_x = old_offset_x + sampling.translations_x[itrans]; RFLOAT offset_y = old_offset_y + sampling.translations_y[itrans]; RFLOAT tdiff2 = 0.; - if ( (!do_helical_refine) || (ignore_helical_symmetry) || (mymodel.data_dim == 3) ) + if ( (!do_helical_refine) || (ignore_helical_symmetry) || (mymodel.data_dim == 3 || mydata.is_tomo) ) tdiff2 += (offset_x - myprior_x) * (offset_x - myprior_x); tdiff2 += (offset_y - myprior_y) * (offset_y - myprior_y); - if (mymodel.data_dim == 3) + if (mymodel.data_dim == 3 || mydata.is_tomo) { RFLOAT offset_z = old_offset_z + sampling.translations_z[itrans]; if ( (!do_helical_refine) || (ignore_helical_symmetry) ) @@ -8021,12 +8022,14 @@ void MlOptimiser::storeWeightedSums(long int part_id, int ibody, if (mymodel.nr_bodies > 1) { Abody = Aori * (mymodel.orient_bodies[ibody]).transpose() * A_rot90 * A * mymodel.orient_bodies[ibody]; - Abody = mydata.obsModel.applyAnisoMag(Abody, optics_group); + if (mydata.is_tomo) Abody = mydata.getRotationMatrix(part_id, img_id) * Abody; + Abody = mydata.obsModel.applyAnisoMag(Abody, optics_group); Abody = mydata.obsModel.applyScaleDifference(Abody, optics_group, mymodel.ori_size, mymodel.pixel_size); } else { - A = mydata.obsModel.applyAnisoMag(A, optics_group); + if (mydata.is_tomo) A = mydata.getRotationMatrix(part_id, img_id) * A; + A = mydata.obsModel.applyAnisoMag(A, optics_group); A = mydata.obsModel.applyScaleDifference(A, optics_group, mymodel.ori_size, mymodel.pixel_size); } @@ -8071,7 +8074,7 @@ void MlOptimiser::storeWeightedSums(long int part_id, int ibody, { myprior_x = XX(exp_prior); myprior_y = YY(exp_prior); - if (mymodel.data_dim == 3) + if (mymodel.data_dim == 3 || mydata.is_tomo) { myprior_z = ZZ(exp_prior); old_offset_z = ZZ(exp_old_offset); @@ -8178,7 +8181,7 @@ void MlOptimiser::storeWeightedSums(long int part_id, int ibody, xshift = oversampled_translations_x[iover_trans]; yshift = oversampled_translations_y[iover_trans]; - if (mymodel.data_dim == 3) + if (mymodel.data_dim == 3 || mydata.is_tomo) zshift = oversampled_translations_z[iover_trans]; RFLOAT rot_deg = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ROT); @@ -8291,7 +8294,7 @@ void MlOptimiser::storeWeightedSums(long int part_id, int ibody, // May18,2015 - Shaoda & Sjors, Helical refinement (translational searches) // Calculate the vector length of myprior RFLOAT mypriors_len2 = myprior_x * myprior_x + myprior_y * myprior_y; - if (mymodel.data_dim == 3) + if (mymodel.data_dim == 3 || mydata.is_tomo) mypriors_len2 += myprior_z * myprior_z; // If it is doing helical refinement AND Cartesian vector myprior has a length > 0, transform the vector to its helical coordinates if ( (do_helical_refine) && (!ignore_helical_symmetry) && (mypriors_len2 > 0.00001) ) @@ -8302,14 +8305,14 @@ void MlOptimiser::storeWeightedSums(long int part_id, int ibody, transformCartesianAndHelicalCoords(myprior_x, myprior_y, myprior_z, myprior_x, myprior_y, myprior_z, rot_deg, tilt_deg, psi_deg, mymodel.data_dim, CART_TO_HELICAL_COORDS); } - if ( (!do_helical_refine) || (ignore_helical_symmetry) || (mymodel.data_dim == 3) ) + if ( (!do_helical_refine) || (ignore_helical_symmetry) || (mymodel.data_dim == 3 || mydata.is_tomo) ) { RFLOAT diffx = myprior_x - old_offset_x - oversampled_translations_x[iover_trans]; thr_wsum_sigma2_offset += weight * my_pixel_size * my_pixel_size * diffx * diffx; } RFLOAT diffy = myprior_y - old_offset_y - oversampled_translations_y[iover_trans]; thr_wsum_sigma2_offset += weight * my_pixel_size * my_pixel_size * diffy * diffy; - if (mymodel.data_dim == 3) + if (mymodel.data_dim == 3 || mydata.is_tomo) { RFLOAT diffz = myprior_z - old_offset_z - oversampled_translations_z[iover_trans]; if ( (!do_helical_refine) || (ignore_helical_symmetry) ) @@ -8447,12 +8450,13 @@ void MlOptimiser::storeWeightedSums(long int part_id, int ibody, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_tilt) = tilt; RFLOAT old_psi = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_psi); DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_psi) = psi; - Matrix1D shifts(mymodel.data_dim); + int shiftdim = (mymodel.data_dim == 3 || mydata.is_tomo) ? 3 : 2 ; + Matrix1D shifts(shiftdim); // include old_offsets for normal refinement (i.e. non multi-body) XX(shifts) = XX(exp_old_offset) + oversampled_translations_x[iover_trans]; YY(shifts) = YY(exp_old_offset) + oversampled_translations_y[iover_trans]; - if (mymodel.data_dim == 3) + if (mymodel.data_dim == 3 || mydata.is_tomo) { ZZ(shifts) = ZZ(exp_old_offset) + oversampled_translations_z[iover_trans]; } @@ -8514,7 +8518,7 @@ void MlOptimiser::storeWeightedSums(long int part_id, int ibody, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_xoff) = XX(shifts); DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_yoff) = YY(shifts); - if (mymodel.data_dim == 3) + if (mymodel.data_dim == 3 || mydata.is_tomo) DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_zoff) = ZZ(shifts); if (ibody == 0) @@ -8793,7 +8797,7 @@ void MlOptimiser::monitorHiddenVariableChanges(long int my_first_part_id, long i mydata.MDbodies[ibody].getValue(EMDL_ORIENT_PSI, old_psi, part_id); mydata.MDbodies[ibody].getValue(EMDL_ORIENT_ORIGIN_X_ANGSTROM, old_xoff, part_id); mydata.MDbodies[ibody].getValue(EMDL_ORIENT_ORIGIN_Y_ANGSTROM, old_yoff, part_id); - if (mymodel.data_dim == 3) + if (mymodel.data_dim == 3 || mydata.is_tomo) { mydata.MDbodies[ibody].getValue(EMDL_ORIENT_ORIGIN_Z_ANGSTROM, old_zoff, part_id); } @@ -8805,7 +8809,7 @@ void MlOptimiser::monitorHiddenVariableChanges(long int my_first_part_id, long i psi = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, 2 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS); xoff = my_pixel_size * DIRECT_A2D_ELEM(exp_metadata, metadata_offset, 3 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS); yoff = my_pixel_size * DIRECT_A2D_ELEM(exp_metadata, metadata_offset, 4 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS); - if (mymodel.data_dim == 3) + if (mymodel.data_dim == 3 || mydata.is_tomo) zoff = my_pixel_size * DIRECT_A2D_ELEM(exp_metadata, metadata_offset, 5 + METADATA_LINE_LENGTH_BEFORE_BODIES + (ibody) * METADATA_NR_BODY_PARAMS); iclass = 0; @@ -8819,7 +8823,7 @@ void MlOptimiser::monitorHiddenVariableChanges(long int my_first_part_id, long i mydata.MDimg.getValue(EMDL_ORIENT_PSI, old_psi, part_id); mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_X_ANGSTROM, old_xoff, part_id); mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_Y_ANGSTROM, old_yoff, part_id); - if (mymodel.data_dim == 3) + if (mymodel.data_dim == 3 || mydata.is_tomo) { mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_Z_ANGSTROM, old_zoff, part_id); } @@ -8831,7 +8835,7 @@ void MlOptimiser::monitorHiddenVariableChanges(long int my_first_part_id, long i psi = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PSI); xoff = my_pixel_size * DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_XOFF); yoff = my_pixel_size * DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_YOFF); - if (mymodel.data_dim == 3) + if (mymodel.data_dim == 3 || mydata.is_tomo) zoff = my_pixel_size * DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ZOFF); iclass = (int)DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_CLASS); @@ -9149,7 +9153,7 @@ void MlOptimiser::calculateExpectedAngularErrors(long int my_first_part_id, long { // Randomly change xoff or yoff RFLOAT ran = rnd_unif(); - if (mymodel.data_dim == 3) + if (mymodel.data_dim == 3 || mydata.is_tomo) { if (ran < 0.3333) xshift = xoff1 + sh_error; @@ -9426,7 +9430,7 @@ void MlOptimiser::updateAngularSampling(bool myverb) RFLOAT new_step = XMIPP_MIN(1.5, 0.75 * acc_trans) * std::pow(2., adaptive_oversampling); // For subtomogram averaging: use at least half times previous step size - if (mymodel.data_dim == 3) // TODO: check: this might just as well work for 2D data... + if (mymodel.data_dim == 3 || mydata.is_tomo) // TODO: check: this might just as well work for 2D data... new_step = XMIPP_MAX(sampling.offset_step / 2., new_step); // Search ranges are five times the last observed changes in offsets @@ -9817,7 +9821,7 @@ void MlOptimiser::setMetaDataSubset(long int first_part_id, long int last_part_i mydata.MDimg.setValue(EMDL_ORIENT_PSI, DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PSI), part_id); mydata.MDimg.setValue(EMDL_ORIENT_ORIGIN_X_ANGSTROM, my_pixel_size * DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_XOFF), part_id); mydata.MDimg.setValue(EMDL_ORIENT_ORIGIN_Y_ANGSTROM, my_pixel_size * DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_YOFF), part_id); - if (mymodel.data_dim == 3) + if (mymodel.data_dim == 3 || mydata.is_tomo) { mydata.MDimg.setValue(EMDL_ORIENT_ORIGIN_Z_ANGSTROM, my_pixel_size * DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ZOFF), part_id); } @@ -9838,7 +9842,7 @@ void MlOptimiser::setMetaDataSubset(long int first_part_id, long int last_part_i { mydata.MDimg.setValue(EMDL_ORIENT_ORIGIN_Y_PRIOR_ANGSTROM, my_pixel_size * prior_y, part_id); } - if (mymodel.data_dim == 3) + if (mymodel.data_dim == 3 || mydata.is_tomo) { RFLOAT prior_z = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ZOFF_PRIOR); if (prior_z < 999.) @@ -9869,7 +9873,7 @@ void MlOptimiser::setMetaDataSubset(long int first_part_id, long int last_part_i mydata.MDbodies[ibody].setValue(EMDL_ORIENT_PSI, psi, part_id); mydata.MDbodies[ibody].setValue(EMDL_ORIENT_ORIGIN_X_ANGSTROM, my_pixel_size * xoff, part_id); mydata.MDbodies[ibody].setValue(EMDL_ORIENT_ORIGIN_Y_ANGSTROM, my_pixel_size * yoff, part_id); - if (mymodel.data_dim == 3) + if (mymodel.data_dim == 3 || mydata.is_tomo) mydata.MDbodies[ibody].setValue(EMDL_ORIENT_ORIGIN_Z_ANGSTROM, my_pixel_size * zoff, part_id); } } @@ -10078,7 +10082,7 @@ void MlOptimiser::getMetaAndImageDataSubset(long int first_part_id, long int las mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_Y_ANGSTROM, yoff_A, part_id); DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_XOFF) = xoff_A / my_pixel_size; DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_YOFF) = yoff_A / my_pixel_size; - if (mymodel.data_dim == 3) + if (mymodel.data_dim == 3 || mydata.is_tomo) { mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_Z_ANGSTROM, zoff_A, part_id); DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ZOFF) = zoff_A / my_pixel_size; @@ -10121,7 +10125,7 @@ void MlOptimiser::getMetaAndImageDataSubset(long int first_part_id, long int las { DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_YOFF_PRIOR) = 999.; } - if (mymodel.data_dim == 3) + if (mymodel.data_dim == 3 || mydata.is_tomo) { if (mydata.MDimg.getValue(EMDL_ORIENT_ORIGIN_Z_PRIOR_ANGSTROM, zoff_A, part_id)) { @@ -10185,7 +10189,7 @@ void MlOptimiser::getMetaAndImageDataSubset(long int first_part_id, long int las mydata.MDbodies[ibody].getValue(EMDL_ORIENT_PSI, psi, part_id); mydata.MDbodies[ibody].getValue(EMDL_ORIENT_ORIGIN_X_ANGSTROM, xoff, part_id); mydata.MDbodies[ibody].getValue(EMDL_ORIENT_ORIGIN_Y_ANGSTROM, yoff, part_id); - if (mymodel.data_dim == 3) + if (mymodel.data_dim == 3 || mydata.is_tomo) mydata.MDbodies[ibody].getValue(EMDL_ORIENT_ORIGIN_Z_ANGSTROM, zoff, part_id); DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_rot) = rot; DIRECT_A2D_ELEM(exp_metadata, metadata_offset, icol_tilt) = tilt; From 389aac4316e410837aa55376201cd097989f4c53 Mon Sep 17 00:00:00 2001 From: scheres Date: Fri, 8 Jul 2022 09:54:49 +0100 Subject: [PATCH 081/495] perform some sanity checks on the tomograms when reading into exp_model --- src/exp_model.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/exp_model.cpp b/src/exp_model.cpp index d8feaee6c..6809bd24d 100644 --- a/src/exp_model.cpp +++ b/src/exp_model.cpp @@ -811,6 +811,7 @@ void Experiment::read(FileName fn_exp, FileName fn_tomo, FileName fn_motion, ObservationModel::loadSafely(fn_exp, obsModel, MDimg, "particles", verb); nr_images_per_optics_group.resize(obsModel.numberOfOpticsGroups(), 0); + std::vector> particles; if (fn_tomo != "") { // For now read in particle table twice: once into MDimg and once into particleSet... @@ -819,6 +820,10 @@ void Experiment::read(FileName fn_exp, FileName fn_tomo, FileName fn_motion, particleSet.read(fn_exp, fn_motion, verb>0); tomogramSet.read(fn_tomo, verb>0); is_tomo = true; + + // For checking tomogram sanity below + particles = particleSet.splitByTomogram(tomogramSet, verb>0); + } // Set is_3D from MDopt @@ -931,6 +936,10 @@ void Experiment::read(FileName fn_exp, FileName fn_tomo, FileName fn_motion, { tomogram = tomogramSet.loadTomogram(tomo_id, false); prev_tomo_name = tomo_name; + + // Tomogram sanity checks + tomogram.validateParticleOptics(particles[tomo_id], particleSet); + particleSet.checkTrajectoryLengths(particles[tomo_id], tomogram.frameCount, "exp_model.read"); } // Add this particle to the Experiment, with its tomogram From 0bbcfb0ec6a915619155959ee65575965ffb3def Mon Sep 17 00:00:00 2001 From: scheres Date: Fri, 8 Jul 2022 09:56:31 +0100 Subject: [PATCH 082/495] repaired error in sanity checks --- src/exp_model.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/exp_model.cpp b/src/exp_model.cpp index 6809bd24d..bd8f87cf2 100644 --- a/src/exp_model.cpp +++ b/src/exp_model.cpp @@ -811,7 +811,7 @@ void Experiment::read(FileName fn_exp, FileName fn_tomo, FileName fn_motion, ObservationModel::loadSafely(fn_exp, obsModel, MDimg, "particles", verb); nr_images_per_optics_group.resize(obsModel.numberOfOpticsGroups(), 0); - std::vector> particles; + std::vector> particles_idx; if (fn_tomo != "") { // For now read in particle table twice: once into MDimg and once into particleSet... @@ -822,7 +822,7 @@ void Experiment::read(FileName fn_exp, FileName fn_tomo, FileName fn_motion, is_tomo = true; // For checking tomogram sanity below - particles = particleSet.splitByTomogram(tomogramSet, verb>0); + particles_idx = particleSet.splitByTomogram(tomogramSet, verb>0); } @@ -938,8 +938,8 @@ void Experiment::read(FileName fn_exp, FileName fn_tomo, FileName fn_motion, prev_tomo_name = tomo_name; // Tomogram sanity checks - tomogram.validateParticleOptics(particles[tomo_id], particleSet); - particleSet.checkTrajectoryLengths(particles[tomo_id], tomogram.frameCount, "exp_model.read"); + tomogram.validateParticleOptics(particles_idx[tomo_id], particleSet); + particleSet.checkTrajectoryLengths(particles_idx[tomo_id], tomogram.frameCount, "exp_model.read"); } // Add this particle to the Experiment, with its tomogram From 2f8a13579928867d562203b8831db3487c0acc8c Mon Sep 17 00:00:00 2001 From: scheres Date: Mon, 11 Jul 2022 13:02:19 +0100 Subject: [PATCH 083/495] first attempt to deal with translations in mydata.is_tomo --- src/exp_model.cpp | 11 ++ src/exp_model.h | 7 +- src/ml_optimiser.cpp | 280 ++++++++++++++------------------------- src/ml_optimiser.h | 6 +- src/ml_optimiser_mpi.cpp | 4 - 5 files changed, 123 insertions(+), 185 deletions(-) diff --git a/src/exp_model.cpp b/src/exp_model.cpp index bd8f87cf2..2fa759caf 100644 --- a/src/exp_model.cpp +++ b/src/exp_model.cpp @@ -93,6 +93,17 @@ Matrix2D Experiment::getRotationMatrix(long int part_id, int img_id) return particles[part_id].images[img_id].Aproj; } +void Experiment::getTranslationInTiltSeries(long int part_id, int img_id, + double shift3d_x, double shift3d_y, double shift3d_z, + double &shift2d_x, double &shift2d_y, double &shift2d_z) +{ + Matrix1D shift2d(3); + Matrix2D Aproj = particles[part_id].images[img_id].Aproj; + shift2d_x = Aproj(0,0) * shift3d_x + Aproj(0,1) * shift3d_y + Aproj(0,2) * shift3d_z; + shift2d_y = Aproj(1,0) * shift3d_x + Aproj(1,1) * shift3d_y + Aproj(2,2) * shift3d_z; + shift2d_z = 0.; +} + void Experiment::getNumberOfImagesPerGroup(std::vector &nr_particles_per_group, int random_subset) { nr_particles_per_group.resize(groups.size()); diff --git a/src/exp_model.h b/src/exp_model.h index b84229d95..a6041f558 100644 --- a/src/exp_model.h +++ b/src/exp_model.h @@ -301,7 +301,12 @@ class Experiment // Get the rotation matrix for Nth image of the subtomo particle Matrix2D getRotationMatrix(long int part_id, int img_id); - // Get the vector of number of images per group_id + // Get the shift in the 2D tilt series image for a subtomo particle from a 3D shift + void getTranslationInTiltSeries(long int part_id, int img_id, + double shift3d_x, double shift3d_y, double shift3d_z, + double &shift2d_x, double &shift2d_y, double &shift2d_z); + + // Get the vector of number of images per group_id void getNumberOfImagesPerGroup(std::vector &nr_particles_per_group, int random_subset = 0); // Get the vector of number of images per group_id diff --git a/src/ml_optimiser.cpp b/src/ml_optimiser.cpp index 214cfe050..e98918277 100644 --- a/src/ml_optimiser.cpp +++ b/src/ml_optimiser.cpp @@ -3222,11 +3222,6 @@ void MlOptimiser::expectation() // E. Check whether everything fits into memory expectationSetupCheckMemory(verb); - // F. Precalculate AB-matrices for on-the-fly shifts - // Use tabulated sine and cosine values instead for 2D helical segments / 3D helical sub-tomogram averaging with on-the-fly shifts - if ( (do_shifts_onthefly) && (!((do_helical_refine) && (!ignore_helical_symmetry))) && !(do_grad && iter > 1)) - precalculateABMatrices(); - #ifdef DEBUG_EXP std::cerr << "Expectation: done setupCheckMemory" << std::endl; @@ -3715,87 +3710,6 @@ void MlOptimiser::expectationSetupCheckMemory(int myverb) } -void MlOptimiser::precalculateABMatrices() -{ - - - - global_fftshifts_ab_coarse.clear(); - global_fftshifts_ab_current.clear(); - global_fftshifts_ab2_coarse.clear(); - global_fftshifts_ab2_current.clear(); - for (int optics_group = 0; optics_group < mydata.numberOfOpticsGroups(); optics_group++) - { - - std::vector > dummy; - global_fftshifts_ab_coarse.push_back(dummy); - global_fftshifts_ab_current.push_back(dummy); - global_fftshifts_ab2_coarse.push_back(dummy); - global_fftshifts_ab2_current.push_back(dummy); - - RFLOAT my_pixel_size = mydata.getOpticsPixelSize(optics_group); - - // Set the global AB-matrices for the FFT phase-shifted images - MultidimArray Fab_current, Fab_coarse; - if (mymodel.data_dim == 3) - Fab_current.resize(image_current_size[optics_group], image_current_size[optics_group], image_current_size[optics_group] / 2 + 1); - else - Fab_current.resize(image_current_size[optics_group], image_current_size[optics_group] / 2 + 1); - long int exp_nr_trans = sampling.NrTranslationalSamplings(); - std::vector oversampled_translations_x, oversampled_translations_y, oversampled_translations_z; - // Note that do_shifts_onthefly is incompatible with do_skip_align because of the loop below - for (long int itrans = 0; itrans < exp_nr_trans; itrans++) - { - // First get the non-oversampled translations as defined by the sampling object - // Feb01,2017 - Shaoda, obsolete, helical reconstuctions never call this function - - // TODO: see how this works with multiple pixel sizes...... - sampling.getTranslationsInPixel(itrans, 0, my_pixel_size, oversampled_translations_x, oversampled_translations_y, oversampled_translations_z, - (do_helical_refine) && (!ignore_helical_symmetry)); // need getTranslations to add random_perturbation - - // Precalculate AB-matrices - RFLOAT tmp_zoff = (mymodel.data_dim == 2) ? (0.) : oversampled_translations_z[0]; - getAbMatricesForShiftImageInFourierTransform(Fab_current, Fab_current, (RFLOAT)image_full_size[optics_group], oversampled_translations_x[0], oversampled_translations_y[0], tmp_zoff); - - windowFourierTransform(Fab_current, Fab_coarse, image_coarse_size[optics_group]); - global_fftshifts_ab_coarse[optics_group].push_back(Fab_coarse); - if (adaptive_oversampling == 0) - { - global_fftshifts_ab_current[optics_group].push_back(Fab_current); - } - else - { - // Then also loop over all its oversampled relatives - // Then loop over all its oversampled relatives - // Feb01,2017 - Shaoda, obsolete, helical reconstuctions never call this function - sampling.getTranslationsInPixel(itrans, adaptive_oversampling, my_pixel_size, oversampled_translations_x, oversampled_translations_y, oversampled_translations_z, - (do_helical_refine) && (!ignore_helical_symmetry)); - for (long int iover_trans = 0; iover_trans < oversampled_translations_x.size(); iover_trans++) - { - // Shift through phase-shifts in the Fourier transform - // Note that the shift search range is centered around (exp_old_xoff, exp_old_yoff) - - RFLOAT tmp_zoff = (mymodel.data_dim == 2) ? (0.) : oversampled_translations_z[iover_trans]; - getAbMatricesForShiftImageInFourierTransform(Fab_current, Fab_current, (RFLOAT)image_full_size[optics_group], oversampled_translations_x[iover_trans], oversampled_translations_y[iover_trans], tmp_zoff); - - global_fftshifts_ab2_current[optics_group].push_back(Fab_current); - if (strict_highres_exp > 0.) - { - windowFourierTransform(Fab_current, Fab_coarse, image_coarse_size[optics_group]); - global_fftshifts_ab2_coarse[optics_group].push_back(Fab_coarse); - } - } - } // end else (adaptive_oversampling == 0) - } // end loop itrans - -#ifdef DEBUG_AB - std::cerr << " global_fftshifts_ab_coarse[optics_group].size()= " << global_fftshifts_ab_coarse[optics_group].size() << " global_fftshifts_ab_current[optics_group].size()= " << global_fftshifts_ab_current[optics_group].size() << std::endl; - std::cerr << " global_fftshifts_ab2_coarse[optics_group].size()= " << global_fftshifts_ab2_coarse[optics_group].size() << " global_fftshifts_ab2_current[optics_group].size()= " << global_fftshifts_ab2_current[optics_group].size() << std::endl; -#endif - } // end loop optics_group - -} - void MlOptimiser::expectationSomeParticles(long int my_first_part_id, long int my_last_part_id) { @@ -5543,7 +5457,6 @@ void MlOptimiser::getFourierTransformsAndCtfs( int shiftdim = (mymodel.data_dim == 3 || mydata.is_tomo) ? 3 : 2 ; my_projected_com.resize(shiftdim); - #ifdef DEBUG_BODIES if (part_id == ROUND(debug1)) { @@ -5696,7 +5609,7 @@ void MlOptimiser::getFourierTransformsAndCtfs( // We do NOT want to accumulate the offsets in the direction along the helix (which is X in the helical coordinate system!) // However, when doing helical local searches, we accumulate offsets // Do NOT accumulate offsets in 3D classification of helices - if ( (!do_skip_align) && (!do_skip_rotate) ) + if (!mydata.is_tomo && (!do_skip_align) && (!do_skip_rotate) ) { // TODO: check whether the following lines make sense bool do_auto_refine_local_searches = (do_auto_refine || do_auto_sampling) && (sampling.healpix_order >= autosampling_hporder_local_searches); @@ -5729,7 +5642,7 @@ void MlOptimiser::getFourierTransformsAndCtfs( #endif } - my_old_offset.selfROUND(); // Below, this rounded my_old_offset will be applied to the actual images + if (!mydata.is_tomo) my_old_offset.selfROUND(); // Below, this rounded my_old_offset will be applied to the actual images #ifdef DEBUG_HELICAL_ORIENTATIONAL_SEARCH if ( (do_helical_refine) && (!ignore_helical_symmetry) ) @@ -5759,7 +5672,10 @@ void MlOptimiser::getFourierTransformsAndCtfs( { // For multi-bodies: store only the old refined offset, not the constant consensus offset or the projected COM of this body if (mymodel.nr_bodies > 1) - exp_old_offset = my_refined_ibody_offset; + if (mydata.is_tomo) // no application of my_old_offset for is_tomo! + exp_old_offset = my_old_offset - my_projected_com + my_refined_ibody_offset; + else + exp_old_offset = my_refined_ibody_offset; else exp_old_offset = my_old_offset; // Not doing helical refinement. Rounded Cartesian offsets are stored. } @@ -5936,15 +5852,22 @@ void MlOptimiser::getFourierTransformsAndCtfs( img() *= mymodel.avg_norm_correction / normcorr; } + if (mydata.is_tomo) + { + // TODO: Need tot take projected 2D shift into account! + // Therefore non-integer shift needs to be done as a FourierTransform phase-shift to prevent another interpolation... + // That's a bit expensive here, but keeps us from changing a lot of code elsewhere... + selfTranslateSubtomoStack2D(img(), my_old_offset, part_id, img_id); + if (has_converged && do_use_reconstruct_images) + selfTranslateSubtomoStack2D(rec_img(), my_old_offset, part_id, img_id); - // TODO! Think this through for 2D stacks.... - if (mydata.is_tomo) REPORT_ERROR("selfTranslate will not work for 2D STA stacks!"); - - selfTranslate(img(), my_old_offset, DONT_WRAP); - if (has_converged && do_use_reconstruct_images) - { - selfTranslate(rec_img(), my_old_offset, DONT_WRAP); - } + } + else + { + selfTranslate(img(), my_old_offset, DONT_WRAP); + if (has_converged && do_use_reconstruct_images) + selfTranslate(rec_img(), my_old_offset, DONT_WRAP); + } //#define DEBUG_SOFTMASK #ifdef DEBUG_SOFTMASK @@ -6303,10 +6226,14 @@ void MlOptimiser::getFourierTransformsAndCtfs( } #endif - // TODO! The below will now work for 2D stacks in 2D, just like selfTranslate() didn't work upwards; Also need to apply per-image rotations, Aproj! - if (mydata.is_tomo) REPORT_ERROR("ERROR: multibody for STA not implemented yet!"); - - shiftImageInFourierTransform(FTo, Faux, (RFLOAT)mymodel.ori_size, + if (mydata.is_tomo) + { + RFLOAT xshift, yshift, zshift; + mydata.getTranslationInTiltSeries(part_id, img_id, XX(other_projected_com), YY(other_projected_com), ZZ(other_projected_com), xshift, yshift, zshift); + shiftImageInFourierTransform(FTo, Faux, (RFLOAT)mymodel.ori_size, xshift, yshift); + } + else + shiftImageInFourierTransform(FTo, Faux, (RFLOAT)mymodel.ori_size, XX(other_projected_com), YY(other_projected_com), (mymodel.data_dim == 3) ? ZZ(other_projected_com) : 0); // Sum the Fourier transforms of all the obodies @@ -6413,7 +6340,7 @@ void MlOptimiser::getFourierTransformsAndCtfs( void MlOptimiser::precalculateShiftedImagesCtfsAndInvSigma2s(bool do_also_unmasked, bool is_for_store_wsums, long int part_id, int exp_current_oversampling, int metadata_offset, int exp_itrans_min, int exp_itrans_max, - std::vector > &exp_Fimg, + std::vector > &exp_Fimg, std::vector > &exp_Fimg_nomask, std::vector > &exp_Fctf, std::vector > > &exp_local_Fimgs_shifted, @@ -6636,9 +6563,11 @@ void MlOptimiser::precalculateShiftedImagesCtfsAndInvSigma2s(bool do_also_unmask #endif } - //// XXXXXXXXXX make 3D shift into 2D shift - - if (mydata.is_tomo) REPORT_ERROR("TODO: think about 2D shifts here!!"); + // For subtomo: convert 3D shifts in the tomogram to 2D shifts in the tilt series images + if (mydata.is_tomo) + { + mydata.getTranslationInTiltSeries(part_id, img_id, xshift, yshift, zshift, xshift, yshift, zshift); + } // Shift through phase-shifts in the Fourier transform // Note that the shift search range is centered around (exp_old_xoff, exp_old_yoff) @@ -6742,7 +6671,7 @@ void MlOptimiser::getAllSquaredDifferences(long int part_id, int ibody, int exp_idir_min, int exp_idir_max, int exp_ipsi_min, int exp_ipsi_max, int exp_itrans_min, int exp_itrans_max, int exp_iclass_min, int exp_iclass_max, RFLOAT &exp_min_diff2, - std::vector &exp_highres_Xi2_img, + std::vector &exp_highres_Xi2_img, std::vector > &exp_Fimg, std::vector > &exp_Fctf, MultidimArray &exp_Mweight, @@ -6992,65 +6921,41 @@ void MlOptimiser::getAllSquaredDifferences(long int part_id, int ibody, { // Calculate shifted image on-the-fly to save replicating memory in multi-threaded jobs. // Feb01,2017 - Shaoda, on-the-fly shifts in helical reconstuctions (2D and 3D) - if ( (do_helical_refine) && (!ignore_helical_symmetry) ) - { - bool use_coarse_size = false; - RFLOAT xshift_in = 0., yshift_in = 0., zshift_in = 0.; - RFLOAT xshift = 0., yshift = 0., zshift = 0.; - - xshift_in = (exp_current_oversampling == 0) ? (oversampled_translations_x[0]) : (oversampled_translations_x[iover_trans]); - yshift_in = (exp_current_oversampling == 0) ? (oversampled_translations_y[0]) : (oversampled_translations_y[iover_trans]); - if (mymodel.data_dim == 3 || mydata.is_tomo) - zshift_in = (exp_current_oversampling == 0) ? (oversampled_translations_z[0]) : (oversampled_translations_z[iover_trans]); - - if (mydata.is_tomo) REPORT_ERROR("ERROR; TODO: think about the below for 2D shifts in stacks..."); - - RFLOAT rot_deg = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ROT); - RFLOAT tilt_deg = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_TILT); - RFLOAT psi_deg = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PSI); - transformCartesianAndHelicalCoords( - xshift_in, yshift_in, zshift_in, - xshift, yshift, zshift, - rot_deg, tilt_deg, psi_deg, - mymodel.data_dim, - HELICAL_TO_CART_COORDS); + bool use_coarse_size = ((exp_current_oversampling == 0) && (YSIZE(Frefctf) == image_coarse_size[optics_group])) + || ((exp_current_oversampling > 0) && (strict_highres_exp > 0.)); + + RFLOAT zshift = 0.; + RFLOAT xshift = (exp_current_oversampling == 0) ? (oversampled_translations_x[0]) : (oversampled_translations_x[iover_trans]); + RFLOAT yshift = (exp_current_oversampling == 0) ? (oversampled_translations_y[0]) : (oversampled_translations_y[iover_trans]); + if (mymodel.data_dim == 3 || mydata.is_tomo) + zshift = (exp_current_oversampling == 0) ? (oversampled_translations_z[0]) : (oversampled_translations_z[iover_trans]); + + if ((do_helical_refine) && (!ignore_helical_symmetry)) + { + RFLOAT rot_deg = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ROT); + RFLOAT tilt_deg = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_TILT); + RFLOAT psi_deg = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PSI); + transformCartesianAndHelicalCoords( + xshift, yshift, zshift, + xshift, yshift, zshift, + rot_deg, tilt_deg, psi_deg, + mymodel.data_dim, + HELICAL_TO_CART_COORDS); + } - use_coarse_size = ((exp_current_oversampling == 0) && (YSIZE(Frefctf) == image_coarse_size[optics_group])) - || ((exp_current_oversampling > 0) && (strict_highres_exp > 0.)); + // For subtomo: convert 3D shifts in the tomogram to 2D shifts in the tilt series images + if (mydata.is_tomo) mydata.getTranslationInTiltSeries(part_id, img_id, + xshift, yshift, zshift, + xshift, yshift, zshift); - if (mydata.is_tomo) REPORT_ERROR("ERROR: TOD: implement what to do with 2D stack translations here!"); + shiftImageInFourierTransformWithTabSincos( + exp_local_Fimgs_shifted[img_id][0], + Fimg_otfshift, + (RFLOAT)mymodel.ori_size, + (use_coarse_size) ? (image_coarse_size[optics_group]) : (image_current_size[optics_group]), + tab_sin, tab_cos, + xshift, yshift, zshift); - shiftImageInFourierTransformWithTabSincos( - exp_local_Fimgs_shifted[img_id][0], - Fimg_otfshift, - (RFLOAT)mymodel.ori_size, - (use_coarse_size) ? (image_coarse_size[optics_group]) : (image_current_size[optics_group]), - tab_sin, tab_cos, - xshift, yshift, zshift); - } - else - { - Complex *myAB; - if (exp_current_oversampling == 0) - { - myAB = (YSIZE(Frefctf) == image_coarse_size[optics_group]) ? global_fftshifts_ab_coarse[optics_group][itrans].data - : global_fftshifts_ab_current[optics_group][itrans].data; - } - else - { - int iitrans = itrans * exp_nr_oversampled_trans + iover_trans; - myAB = (strict_highres_exp > 0.) ? global_fftshifts_ab2_coarse[optics_group][iitrans].data - : global_fftshifts_ab2_current[optics_group][iitrans].data; - } - FOR_ALL_DIRECT_ELEMENTS_IN_MULTIDIMARRAY(exp_local_Fimgs_shifted[img_id][0]) - { - RFLOAT real = (*(myAB + n)).real * (DIRECT_MULTIDIM_ELEM(exp_local_Fimgs_shifted[img_id][0], n)).real - - (*(myAB + n)).imag *(DIRECT_MULTIDIM_ELEM(exp_local_Fimgs_shifted[img_id][0], n)).imag; - RFLOAT imag = (*(myAB + n)).real * (DIRECT_MULTIDIM_ELEM(exp_local_Fimgs_shifted[img_id][0], n)).imag - + (*(myAB + n)).imag *(DIRECT_MULTIDIM_ELEM(exp_local_Fimgs_shifted[img_id][0], n)).real; - DIRECT_MULTIDIM_ELEM(Fimg_otfshift, n) = Complex(real, imag); - } - } Fimg_shift = Fimg_otfshift.data; } #ifdef TIMING @@ -8175,24 +8080,31 @@ void MlOptimiser::storeWeightedSums(long int part_id, int ibody, else { // Feb01,2017 - Shaoda, on-the-fly shifts in helical reconstuctions (2D and 3D) - if ( (do_helical_refine) && (!ignore_helical_symmetry) ) + if ( mydata.is_tomo || ((do_helical_refine) && (!ignore_helical_symmetry)) ) { - RFLOAT xshift = 0., yshift = 0., zshift = 0.; - - xshift = oversampled_translations_x[iover_trans]; - yshift = oversampled_translations_y[iover_trans]; + RFLOAT zshift = 0.; + RFLOAT xshift = oversampled_translations_x[iover_trans]; + RFLOAT yshift = oversampled_translations_y[iover_trans]; if (mymodel.data_dim == 3 || mydata.is_tomo) zshift = oversampled_translations_z[iover_trans]; - RFLOAT rot_deg = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ROT); - RFLOAT tilt_deg = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_TILT); - RFLOAT psi_deg = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PSI); - transformCartesianAndHelicalCoords( - xshift, yshift, zshift, - xshift, yshift, zshift, - rot_deg, tilt_deg, psi_deg, - mymodel.data_dim, - HELICAL_TO_CART_COORDS); + if ((do_helical_refine) && (!ignore_helical_symmetry)) + { + RFLOAT rot_deg = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ROT); + RFLOAT tilt_deg = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_TILT); + RFLOAT psi_deg = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PSI); + transformCartesianAndHelicalCoords( + xshift, yshift, zshift, + xshift, yshift, zshift, + rot_deg, tilt_deg, psi_deg, + mymodel.data_dim, + HELICAL_TO_CART_COORDS); + } + + // For subtomo: convert 3D shifts in the tomogram to 2D shifts in the tilt series images + if (mydata.is_tomo) mydata.getTranslationInTiltSeries(part_id, img_id, + xshift, yshift, zshift, + xshift, yshift, zshift); // Fimg_shift shiftImageInFourierTransformWithTabSincos( @@ -10357,3 +10269,17 @@ void MlOptimiser::applySubtomoCorrection(MultidimArray &Fimg, Multidim FstMulti.clear(); } } + +void MlOptimiser::selfTranslateSubtomoStack2D(MultidimArray &img, const Matrix1D &v, long int part_id, int img_id) +{ + RFLOAT xshift, yshift, zshift; + mydata.getTranslationInTiltSeries(part_id, img_id, XX(v), YY(v), ZZ(v), xshift, yshift, zshift); + + FourierTransformer transformer; + MultidimArray FT, Faux; + transformer.FourierTransform(img, FT, false); + Faux = FT; + shiftImageInFourierTransform(Faux, FT, XSIZE(img), (RFLOAT)mymodel.ori_size, xshift, yshift); + transformer.inverseFourierTransform(FT, img); + +} \ No newline at end of file diff --git a/src/ml_optimiser.h b/src/ml_optimiser.h index bba74f31c..c6317d3b6 100644 --- a/src/ml_optimiser.h +++ b/src/ml_optimiser.h @@ -979,9 +979,6 @@ class MlOptimiser /* Check whether everything fits into memory, possibly adjust nr_pool and setup thread task managers */ void expectationSetupCheckMemory(int myverb = 1); - /* For on-the-fly shifts, precalculates AB-matrices */ - void precalculateABMatrices(); - /* Perform the expectation integration over all k, phi and series elements for a number (some) of pooled particles * The number of pooled particles is determined by max_nr_pool and some memory checks in expectationSetup() */ @@ -1190,6 +1187,9 @@ class MlOptimiser void applySubtomoCorrection(MultidimArray &Fimg, MultidimArray &Fimg_nomask , MultidimArray &Fctf, MultidimArray &FstMulti); + // Use phase-shifts in FourierTransform to prevent another interpolation for projected (non-integer) shifts in 2D subtomo stacks + void selfTranslateSubtomoStack2D(MultidimArray &I1, const Matrix1D &v, long int part_id, int img_id); + }; // Global call to threaded core of doThreadExpectationSomeParticles diff --git a/src/ml_optimiser_mpi.cpp b/src/ml_optimiser_mpi.cpp index de6e014ff..74274e8f9 100644 --- a/src/ml_optimiser_mpi.cpp +++ b/src/ml_optimiser_mpi.cpp @@ -869,10 +869,6 @@ void MlOptimiserMpi::expectation() int myverb = (node->rank == first_follower) ? 1 : 0; MlOptimiser::expectationSetupCheckMemory(myverb); - // F. Precalculate AB-matrices for on-the-fly shifts - // Use tabulated sine and cosine values instead for 2D helical segments / 3D helical sub-tomogram averaging with on-the-fly shifts - if ( (do_shifts_onthefly) && (!((do_helical_refine) && (!ignore_helical_symmetry))) && !(do_grad && iter > 1)) - precalculateABMatrices(); } // Follower 1 sends has_converged to everyone else (in particular the leader needs it!) node->relion_MPI_Bcast(&has_converged, 1, MPI_INT, first_follower, MPI_COMM_WORLD); From e2bb879fb5aa86ce09791b182af1c24d055c9c91 Mon Sep 17 00:00:00 2001 From: scheres Date: Mon, 11 Jul 2022 16:29:53 +0100 Subject: [PATCH 084/495] first part of debugging --- src/exp_model.cpp | 11 +++++-- src/jaz/tomography/particle_set.cpp | 6 ++-- src/jaz/tomography/particle_set.h | 2 +- src/jaz/tomography/programs/subtomo.cpp | 3 +- src/jaz/tomography/tomogram_set.cpp | 38 ++++++++++----------- src/jaz/tomography/tomogram_set.h | 2 +- src/ml_optimiser.cpp | 44 ++++++++++++++++++------- 7 files changed, 65 insertions(+), 41 deletions(-) diff --git a/src/exp_model.cpp b/src/exp_model.cpp index 2fa759caf..db8f1cb72 100644 --- a/src/exp_model.cpp +++ b/src/exp_model.cpp @@ -817,7 +817,12 @@ void Experiment::read(FileName fn_exp, FileName fn_tomo, FileName fn_motion, } else { - // MDimg and MDopt have to be read at the same time, so that the optics groups can be + MetaDataTable MDgen; + MDgen.read(fn_exp, "general"); + if (MDgen.numberOfObjects() > 0) + MDgen.getValue(EMDL_TOMO_SUBTOMOGRAM_STACK2D, is_tomo); + + // MDimg and MDopt have to be read at the same time, so that the optics groups can be // renamed in case they are non-contiguous or not sorted ObservationModel::loadSafely(fn_exp, obsModel, MDimg, "particles", verb); nr_images_per_optics_group.resize(obsModel.numberOfOpticsGroups(), 0); @@ -828,8 +833,8 @@ void Experiment::read(FileName fn_exp, FileName fn_tomo, FileName fn_motion, // For now read in particle table twice: once into MDimg and once into particleSet... // TODO: work with pointer to avoid duplication? // TODO: ObservationModel::loadSafely(fn_exp, obsModel, MDimg, "particles", verb, true, true); - particleSet.read(fn_exp, fn_motion, verb>0); tomogramSet.read(fn_tomo, verb>0); + particleSet.read(fn_exp, fn_motion, verb>0); is_tomo = true; // For checking tomogram sanity below @@ -973,7 +978,7 @@ void Experiment::read(FileName fn_exp, FileName fn_tomo, FileName fn_motion, double dz_pos = tomogram.getDepthOffset(f, pos); double dz = tomogram.handedness * tomogram.optics.pixelSize * tomogram.defocusSlope * dz_pos; - addImageToParticle(img_name, part_id, group_id, optics_group, &P, dz); + addImageToParticle(my_name, part_id, group_id, optics_group, &P, dz); } } diff --git a/src/jaz/tomography/particle_set.cpp b/src/jaz/tomography/particle_set.cpp index d92546bbc..a173dd4dd 100644 --- a/src/jaz/tomography/particle_set.cpp +++ b/src/jaz/tomography/particle_set.cpp @@ -148,7 +148,7 @@ std::vector > ParticleSet::splitByTomogram(const Tomo std::map name_to_index; - for (int t = 0; t < tc; t++) + for (int t = 0; t < tc; t++) { const std::string name = tomogramSet.globalTable.getString(EMDL_TOMO_NAME, t); name_to_index[name] = t; @@ -419,10 +419,12 @@ void ParticleSet::shiftParticleBy(ParticleIndex particle_id, gravis::d3Vector sh partTable.setValue(EMDL_IMAGE_COORD_Z, z + shift.z, particle_id.value); } -void ParticleSet::write(const std::string& filename) const +void ParticleSet::write(const std::string& filename) { std::ofstream ofs(filename); + genTable.setName("general"); + genTable.setValue(EMDL_TOMO_SUBTOMOGRAM_STACK2D, is_stack2d); genTable.write(ofs); optTable.write(ofs); partTable.write(ofs); diff --git a/src/jaz/tomography/particle_set.h b/src/jaz/tomography/particle_set.h index 6ac8f6adb..796e0600b 100644 --- a/src/jaz/tomography/particle_set.h +++ b/src/jaz/tomography/particle_set.h @@ -57,7 +57,7 @@ class ParticleSet void moveParticleTo(ParticleIndex particle_id, gravis::d3Vector pos); void shiftParticleBy(ParticleIndex particle_id, gravis::d3Vector shift); - void write(const std::string& filename) const; + void write(const std::string& filename); void writeTrajectories(const std::string& filename) const; void setImageFileNames(std::string data, std::string weight, ParticleIndex particle_id); diff --git a/src/jaz/tomography/programs/subtomo.cpp b/src/jaz/tomography/programs/subtomo.cpp index 3b627c2dd..d4a874c6f 100644 --- a/src/jaz/tomography/programs/subtomo.cpp +++ b/src/jaz/tomography/programs/subtomo.cpp @@ -375,7 +375,8 @@ void SubtomoProgram::writeParticleSet( const double ps_out = binning * ps_img; copy.optTable.setValue(EMDL_OPTIMISER_DATA_ARE_CTF_PREMULTIPLIED, true, og); - copy.optTable.setValue(EMDL_IMAGE_DIMENSIONALITY, 3, og); + int datadim = (do_stack2d) ? 2 : 3; + copy.optTable.setValue(EMDL_IMAGE_DIMENSIONALITY, datadim, og); copy.optTable.setValue(EMDL_TOMO_SUBTOMOGRAM_BINNING, binning, og); copy.optTable.setValue(EMDL_IMAGE_PIXEL_SIZE, ps_out, og); copy.optTable.setValue(EMDL_IMAGE_SIZE, cropSize, og); diff --git a/src/jaz/tomography/tomogram_set.cpp b/src/jaz/tomography/tomogram_set.cpp index 4545507b2..17a65f953 100644 --- a/src/jaz/tomography/tomogram_set.cpp +++ b/src/jaz/tomography/tomogram_set.cpp @@ -20,8 +20,23 @@ TomogramSet::TomogramSet() TomogramSet::TomogramSet(FileName filename, bool verbose) { - if (!read(filename, verbose)) + bool success = read(filename, verbose); +} + +bool TomogramSet::read(FileName filename, bool verbose) +{ + + globalTable.read(filename, "global"); + globalTable.setName("global"); + + const int tc = globalTable.numberOfObjects(); + + if (tc == 0 || !globalTable.containsLabel(EMDL_TOMO_TILT_SERIES_STARFILE)) { + std::cerr << "Warning: " << filename + << " does not have rlnTomoTiltSeriesStarFile labels. It may be written in an old format. If so, will try to convert ..." + << std::endl; + // This may be a tomograms.star file in the old, original relion-4.0 format. Try to convert FileName mydir = filename.beforeLastOf("/"); @@ -77,29 +92,10 @@ TomogramSet::TomogramSet(FileName filename, bool verbose) } } - } -} - -bool TomogramSet::read(std::string filename, bool verbose) -{ - - globalTable.read(filename, "global"); - globalTable.setName("global"); - - const int tc = globalTable.numberOfObjects(); - - if (tc == 0) return false; - - if (!globalTable.containsLabel(EMDL_TOMO_TILT_SERIES_STARFILE)) - { - std::cerr << "Warning: " << filename - << " does not have rlnTomoTiltSeriesStarFile labels. It may be written in an old format. If so, will try to convert ..." - << std::endl; - return false; - } + // Continue reading with the new way of tomograms if (!globalTable.containsLabel(EMDL_TOMO_NAME)) { REPORT_ERROR("ERROR: input starfile for TomogramSet " + filename + " does not contain rlnTomoName label "); diff --git a/src/jaz/tomography/tomogram_set.h b/src/jaz/tomography/tomogram_set.h index 8af0b7a64..fd108db73 100644 --- a/src/jaz/tomography/tomogram_set.h +++ b/src/jaz/tomography/tomogram_set.h @@ -20,7 +20,7 @@ class TomogramSet TomogramSet(FileName filename, bool verbose = true); // return false if this is not a TomogramSet - bool read(std::string filename, bool verbose = true); + bool read(FileName filename, bool verbose = true); void write(FileName filename); Tomogram loadTomogram(int index, bool loadImageData) const; diff --git a/src/ml_optimiser.cpp b/src/ml_optimiser.cpp index e98918277..f3d8ec89d 100644 --- a/src/ml_optimiser.cpp +++ b/src/ml_optimiser.cpp @@ -2457,6 +2457,8 @@ void MlOptimiser::calculateSumOfPowerSpectraAndAverageImage(MultidimArray A; Euler_angles2matrix(rot, tilt, psi, A, true); + if (mydata.is_tomo) A = mydata.getRotationMatrix(part_id, img_id) * A; + // At this point anisotropic magnification shouldn't matter // Also: dont applyScaleDifference, as img() was rescaled to mymodel.ori_size and mymodel.pixel_size //A = mydata.obsModel.applyAnisoMag(A, optics_group); @@ -2700,7 +2704,15 @@ void MlOptimiser::calculateSumOfPowerSpectraAndAverageImage(MultidimArray 1) - { - REPORT_ERROR("ERROR: calculateExpectedAngularErrors will not work for multiple images per particle from 3.1..."); - } } // Separate angular error estimate for each of the classes @@ -8839,7 +8844,7 @@ void MlOptimiser::calculateExpectedAngularErrors(long int my_first_part_id, long std::cout << " Estimating accuracies in the orientational assignment ... " << std::endl; int nr_particles = (my_last_part_id - my_first_part_id + 1); - init_progress_bar( nr_particles * mymodel.nr_classes * mymodel.nr_bodies); + init_progress_bar( n_trials * mymodel.nr_classes * mymodel.nr_bodies); for (int iclass = 0; iclass < mymodel.nr_classes * mymodel.nr_bodies; iclass++) { @@ -8947,6 +8952,12 @@ void MlOptimiser::calculateExpectedAngularErrors(long int my_first_part_id, long DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_CTF_KFACTOR), DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_CTF_PHASE_SHIFT)); + if (mydata.is_tomo) + { + ctf.DeltafU += mydata.particles[part_id].images[img_id].dz; + ctf.DeltafV += mydata.particles[part_id].images[img_id].dz; + } + ctf.getFftwImage(Fctf, image_full_size[optics_group], image_full_size[optics_group], my_pixel_size, ctf_phase_flipped, only_flip_phases, intact_ctf_first_peak, true, do_ctf_padding); @@ -9030,6 +9041,8 @@ void MlOptimiser::calculateExpectedAngularErrors(long int my_first_part_id, long // Get the FT of the first image Euler_angles2matrix(rot1, tilt1, psi1, A1, false); + + if (mydata.is_tomo) A1 = mydata.getRotationMatrix(part_id, img_id) * A1; A1 = mydata.obsModel.applyAnisoMag(A1, optics_group); A1 = mydata.obsModel.applyScaleDifference(A1, optics_group, mymodel.ori_size, mymodel.pixel_size); (mymodel.PPref[iclass]).get2DFourierTransform(F1, A1); @@ -9092,13 +9105,20 @@ void MlOptimiser::calculateExpectedAngularErrors(long int my_first_part_id, long { // Get new rotated version of reference Euler_angles2matrix(rot2, tilt2, psi2, A2, false); + if (mydata.is_tomo) A2 = mydata.getRotationMatrix(part_id, img_id) * A2; A2 = mydata.obsModel.applyAnisoMag(A2, optics_group); A2 = mydata.obsModel.applyScaleDifference(A2, optics_group, mymodel.ori_size, mymodel.pixel_size); (mymodel.PPref[iclass]).get2DFourierTransform(F2, A2); } else { - // Get shifted version + if (mydata.is_tomo) + { + RFLOAT xshift, yshift, zshift; + mydata.getTranslationInTiltSeries(part_id, img_id, xshift, yshift, zshift, xshift, yshift, zshift); + } + + // Get shifted version shiftImageInFourierTransform(F1, F2, (RFLOAT)image_full_size[optics_group], -xshift, -yshift, -zshift); } @@ -9869,7 +9889,7 @@ void MlOptimiser::getMetaAndImageDataSubset(long int first_part_id, long int las // Get the image names from the MDimg table FileName fn_img="", fn_rec_img="", fn_ctf=""; if (!mydata.getImageNameOnScratch(part_id, img_id, fn_img)) - mydata.MDimg.getValue(EMDL_IMAGE_NAME, fn_img, part_id); + fn_img = mydata.particles[part_id].images[img_id].name; if (mymodel.data_dim == 3 && do_ctf_correction) { From 7fdee68d6ae467cd21a66b0bb212c27def363f93 Mon Sep 17 00:00:00 2001 From: scheres Date: Tue, 12 Jul 2022 10:19:10 +0100 Subject: [PATCH 085/495] repaired bug in image filenames --- src/exp_model.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/exp_model.cpp b/src/exp_model.cpp index db8f1cb72..96d29f8b2 100644 --- a/src/exp_model.cpp +++ b/src/exp_model.cpp @@ -971,7 +971,7 @@ void Experiment::read(FileName fn_exp, FileName fn_tomo, FileName fn_motion, for (int f = 0; f < fc; f++) { - FileName my_name = integerToString(f) + "@" + img_name; + FileName my_name = integerToString(f+1) + "@" + img_name; d4Matrix P = tomogram.projectionMatrices[f] * d4Matrix(A); From b2063226570b189b23e8e80cc02eecd89478f6c6 Mon Sep 17 00:00:00 2001 From: scheres Date: Tue, 12 Jul 2022 12:42:00 +0100 Subject: [PATCH 086/495] debugging, now runs with skip_align --- src/acc/acc_ml_optimiser_impl.h | 22 +++----- src/exp_model.cpp | 19 +++++-- src/exp_model.h | 14 +++-- src/jaz/tomography/programs/subtomo.cpp | 2 +- src/ml_optimiser.cpp | 73 +++++++++++-------------- 5 files changed, 62 insertions(+), 68 deletions(-) diff --git a/src/acc/acc_ml_optimiser_impl.h b/src/acc/acc_ml_optimiser_impl.h index 588aa4d29..906898127 100644 --- a/src/acc/acc_ml_optimiser_impl.h +++ b/src/acc/acc_ml_optimiser_impl.h @@ -719,21 +719,13 @@ void getFourierTransformsAndCtfs(long int part_id, CTIC(accMLO->timer,"CTFRead2D"); CTF ctf; ctf.setValuesByGroup( - &(baseMLO->mydata).obsModel, optics_group, - DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_CTF_DEFOCUS_U), - DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_CTF_DEFOCUS_V), - DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_CTF_DEFOCUS_ANGLE), - DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_CTF_BFACTOR), - DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_CTF_KFACTOR), - DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_CTF_PHASE_SHIFT)); - - // Apply the dz to the defocus for 2D image stacks in STA - if (baseMLO->mydata.is_tomo) - { - ctf.DeltafU += baseMLO->mydata.particles[part_id].images[img_id].dz; - ctf.DeltafV += baseMLO->mydata.particles[part_id].images[img_id].dz; - } - + &(baseMLO->mydata).obsModel, optics_group, + (baseMLO->mydata.is_tomo) ? baseMLO->mydata.particles[part_id].images[img_id].defU : DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_CTF_DEFOCUS_U), + (baseMLO->mydata.is_tomo) ? baseMLO->mydata.particles[part_id].images[img_id].defV : DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_CTF_DEFOCUS_V), + (baseMLO->mydata.is_tomo) ? baseMLO->mydata.particles[part_id].images[img_id].defAngle : DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_CTF_DEFOCUS_ANGLE), + DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_CTF_BFACTOR), + DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_CTF_KFACTOR), + DIRECT_A2D_ELEM(baseMLO->exp_metadata, op.metadata_offset, METADATA_CTF_PHASE_SHIFT)); ctf.getFftwImage(Fctf, baseMLO->image_full_size[optics_group], baseMLO->image_full_size[optics_group], my_pixel_size, baseMLO->ctf_phase_flipped, baseMLO->only_flip_phases, baseMLO->intact_ctf_first_peak, true, baseMLO->do_ctf_padding); diff --git a/src/exp_model.cpp b/src/exp_model.cpp index 96d29f8b2..ba6b01a49 100644 --- a/src/exp_model.cpp +++ b/src/exp_model.cpp @@ -175,7 +175,7 @@ void Experiment::addParticle(int random_subset, int tomogram_id) } void Experiment::addImageToParticle(std::string img_name, long int part_id, long int group_id, int optics_group, - d4Matrix *Aproj, RFLOAT dz) + d4Matrix *Aproj, CTF *ctf) { if (group_id >= groups.size()) REPORT_ERROR("Experiment::addImageToParticle: group_id out of range"); @@ -196,12 +196,22 @@ void Experiment::addImageToParticle(std::string img_name, long int part_id, long } ExpImage img; + if (ctf == NULL) + { + img.defU = img.defV = img.defAngle = 0.; + } + else + { + img.defU = ctf->DeltafU; + img.defV = ctf->DeltafV; + img.defAngle = ctf->azimuthal_angle; + } + img.name = img_name; img.particle_id = part_id; img.group_id = group_id; img.optics_group = optics_group; img.Aproj = A; - img.dz = dz; nr_images_per_optics_group[optics_group]++; img.optics_group_id = nr_images_per_optics_group[optics_group] - 1; @@ -975,10 +985,9 @@ void Experiment::read(FileName fn_exp, FileName fn_tomo, FileName fn_motion, d4Matrix P = tomogram.projectionMatrices[f] * d4Matrix(A); - double dz_pos = tomogram.getDepthOffset(f, pos); - double dz = tomogram.handedness * tomogram.optics.pixelSize * tomogram.defocusSlope * dz_pos; + CTF ctf = tomogram.getCtf(f, pos); - addImageToParticle(my_name, part_id, group_id, optics_group, &P, dz); + addImageToParticle(my_name, part_id, group_id, optics_group, &P, &ctf); } } diff --git a/src/exp_model.h b/src/exp_model.h index a6041f558..68562fac0 100644 --- a/src/exp_model.h +++ b/src/exp_model.h @@ -64,8 +64,8 @@ class ExpImage // Projection matrix for tilt series stacks Matrix2D Aproj; - // Delta-Z for defocus adjustment of tilt seriers - RFLOAT dz; + // CTF information for defocus adjustment of tilt seriers + RFLOAT defU, defV, defAngle; // Empty Constructor ExpImage() {} @@ -83,7 +83,9 @@ class ExpImage optics_group = copy.optics_group; img = copy.img; Aproj = copy.Aproj; - dz = copy.dz; + defU = copy.defU; + defV = copy.defV; + defAngle = copy.defAngle; } // Define assignment operator in terms of the copy constructor @@ -97,7 +99,9 @@ class ExpImage optics_group = copy.optics_group; img = copy.img; Aproj = copy.Aproj; - dz = copy.dz; + defU = copy.defU; + defV = copy.defV; + defAngle = copy.defAngle; return *this; } }; @@ -322,7 +326,7 @@ class Experiment void addParticle(int random_subset = 0, int tomogram_id = 0); // Add an image to the given particle - void addImageToParticle(std::string img_name, long int part_id, long int group_id, int optics_group, d4Matrix *Aproj = NULL, RFLOAT dz = 0.); + void addImageToParticle(std::string img_name, long int part_id, long int group_id, int optics_group, d4Matrix *Aproj = NULL, CTF *ctf = NULL); // Add a group long int addGroup(std::string mic_name, int optics_group); diff --git a/src/jaz/tomography/programs/subtomo.cpp b/src/jaz/tomography/programs/subtomo.cpp index d4a874c6f..1d469257e 100644 --- a/src/jaz/tomography/programs/subtomo.cpp +++ b/src/jaz/tomography/programs/subtomo.cpp @@ -374,7 +374,7 @@ void SubtomoProgram::writeParticleSet( const double ps_img = copy.optTable.getDouble(EMDL_TOMO_TILT_SERIES_PIXEL_SIZE, og); const double ps_out = binning * ps_img; - copy.optTable.setValue(EMDL_OPTIMISER_DATA_ARE_CTF_PREMULTIPLIED, true, og); + copy.optTable.setValue(EMDL_OPTIMISER_DATA_ARE_CTF_PREMULTIPLIED, !do_stack2d, og); int datadim = (do_stack2d) ? 2 : 3; copy.optTable.setValue(EMDL_IMAGE_DIMENSIONALITY, datadim, og); copy.optTable.setValue(EMDL_TOMO_SUBTOMOGRAM_BINNING, binning, og); diff --git a/src/ml_optimiser.cpp b/src/ml_optimiser.cpp index f3d8ec89d..05b6c9397 100644 --- a/src/ml_optimiser.cpp +++ b/src/ml_optimiser.cpp @@ -2708,8 +2708,10 @@ void MlOptimiser::calculateSumOfPowerSpectraAndAverageImage(MultidimArray 0) ? YSIZE(exp_local_Fctf[0]) != exp_current_image_size : true; // size has changed + bool do_ctf_invsig = (exp_local_Fctf.size() > 0) ? YSIZE(exp_local_Fctf[img_id]) != exp_current_image_size : true; // size has changed bool do_masked_shifts = (do_ctf_invsig || nr_shifts != exp_local_Fimgs_shifted[img_id].size()); // size or nr_shifts has changed if (do_masked_shifts) @@ -6802,7 +6798,8 @@ void MlOptimiser::getAllSquaredDifferences(long int part_id, int ibody, // if so, proceed with projecting the reference in that direction bool do_proceed = (exp_ipass==0) ? true : isSignificantAnyImageAnyTranslation(iorientclass, exp_itrans_min, exp_itrans_max, exp_Mcoarse_significant); - if (do_proceed && pdf_orientation > 0.) + + if (do_proceed && pdf_orientation > 0.) { // Now get the oversampled (rot, tilt, psi) triplets // This will be only the original (rot,tilt,psi) triplet in the first pass (exp_current_oversampling==0) @@ -6815,7 +6812,6 @@ void MlOptimiser::getAllSquaredDifferences(long int part_id, int ibody, // loop over all images inside this particle for (int img_id = 0; img_id < exp_nr_images; img_id++) { - RFLOAT my_pixel_size = mydata.getImagePixelSize(part_id, img_id); int optics_group = mydata.getOpticsGroup(part_id, img_id); @@ -6849,7 +6845,6 @@ void MlOptimiser::getAllSquaredDifferences(long int part_id, int ibody, (mymodel.PPref[exp_iclass]).get2DFourierTransform(Fref, A); } - #ifdef TIMING // Only time one thread, as I also only time one MPI process if (part_id == mydata.sorted_idx[exp_my_first_part_id]) @@ -6906,15 +6901,12 @@ void MlOptimiser::getAllSquaredDifferences(long int part_id, int ibody, if (part_id == mydata.sorted_idx[exp_my_first_part_id]) timer.tic(TIMING_DIFF2_GETSHIFT); #endif - /// Now get the shifted image + /// Now get the shifted image // Use a pointer to avoid copying the entire array again in this highly expensive loop Complex *Fimg_shift; if (!do_shifts_onthefly) { - long int ishift = img_id * exp_nr_oversampled_trans * exp_nr_images + - (itrans - exp_itrans_min) * exp_nr_oversampled_trans + iover_trans; - if (do_skip_align) - ishift = img_id; + long int ishift = (do_skip_align) ? 0 : (itrans - exp_itrans_min) * exp_nr_oversampled_trans + iover_trans; #ifdef DEBUG_CHECKSIZES if (ishift >= exp_local_Fimgs_shifted.size()) { @@ -7074,15 +7066,20 @@ void MlOptimiser::getAllSquaredDifferences(long int part_id, int ibody, //fnt.compose("Fref1_i", ihidden_over, "spi"); tt.write(fnt); + tt().resize(exp_local_Fctf[img_id]); + FOR_ALL_DIRECT_ELEMENTS_IN_MULTIDIMARRAY(exp_local_Fctf[img_id]) + { + DIRECT_MULTIDIM_ELEM(tt(), n) = DIRECT_MULTIDIM_ELEM(exp_local_Fctf[img_id], n); + } + tt.write("ctf.spi"); - //for (int i = 0; i< mymodel.scale_correction.size(); i++) + //for (int i = 0; i< mymodel.scale_correction.size(); i++) // std::cerr << i << " scale="< allTables = MetaDataTable::readAll(ifs, tc+1); for (int t = 0; t < tc; t++) { @@ -73,48 +67,32 @@ bool TomogramSet::read(FileName filename, bool verbose) tomogramTables[t] = allTables[t+1]; tomogramTables[t].setName(expectedNewName); - // Check there is a rlnTomoTiltMovieIndex label, otherwise add one - if (!tomogramTables[t].containsLabel(EMDL_MICROGRAPH_PRE_EXPOSURE) ) - { - REPORT_ERROR("ERROR: tilt series " + expectedNewName + " does not contain compulsory rlnMicrographPreExposure label"); - } - - // As this is a conversion, also save already all the tilt series starfiles in a new directory - // Create output directory if necessary - FileName fn_star = mydir + "/tilt_series/" + expectedNewName + ".star"; - globalTable.setValue(EMDL_TOMO_TILT_SERIES_STARFILE, fn_star, t); - FileName newdir = fn_star.beforeLastOf("/"); - if (!exists(newdir)) mktree(newdir); - - // Write the individual tomogram starfile - tomogramTables[t].newSort(EMDL_MICROGRAPH_PRE_EXPOSURE); - tomogramTables[t].write(fn_star); - } } } - - - // Continue reading with the new way of tomograms - if (!globalTable.containsLabel(EMDL_TOMO_NAME)) - { - REPORT_ERROR("ERROR: input starfile for TomogramSet " + filename + " does not contain rlnTomoName label "); - } - - tomogramTables.resize(tc); - - for (int t = 0; t < tc; t++) + else { - FileName name = globalTable.getString(EMDL_TOMO_NAME, t); - std::string fn_star = globalTable.getString(EMDL_TOMO_TILT_SERIES_STARFILE, t); - tomogramTables[t].read(fn_star, name); - // Make sure tilt images are sorted on their pre-exposure (used to convert back from single large metadatatable) - if (tomogramTables[t].containsLabel(EMDL_MICROGRAPH_PRE_EXPOSURE)) - tomogramTables[t].newSort(EMDL_MICROGRAPH_PRE_EXPOSURE); - else - REPORT_ERROR("ERROR: tomogramTable does not contain compulsory rlnMicrographPreExposure label"); + // The new way of reading in the tomogram STAR files + for (int t = 0; t < tc; t++) + { + FileName name = globalTable.getString(EMDL_TOMO_NAME, t); + std::string fn_star = globalTable.getString(EMDL_TOMO_TILT_SERIES_STARFILE, t); + std::cerr << " fn_star= " << fn_star << std::endl; + tomogramTables[t].read(fn_star, name); + + if (!tomogramTables[t].containsLabel(EMDL_MICROGRAPH_PRE_EXPOSURE)) + REPORT_ERROR("ERROR: tomogramTable does not contain compulsory rlnMicrographPreExposure label"); + + // Make sure tilt images are again sorted in their original order when in convertBackFromSingleMetaDataTable() below, + // converting back from single large metadatatable for ctffind and motioncorr runners + int i = 0; + FOR_ALL_OBJECTS_IN_METADATA_TABLE(tomogramTables[t]) + { + tomogramTables[t].setValue(EMDL_TOMO_TILT_MOVIE_INDEX, i); + i++; + } + } } - return true; } @@ -137,6 +115,8 @@ void TomogramSet::write(FileName filename) FileName newdir = fn_newstar.beforeLastOf("/"); if (!exists(newdir)) mktree(newdir); + // Deactivate the label for sorting the tilt series images + tomogramTables[t].deactivateLabel(EMDL_TOMO_TILT_MOVIE_INDEX); // Write the individual tomogram starfile tomogramTables[t].write(fn_newstar); } @@ -172,7 +152,6 @@ Tomogram TomogramSet::loadTomogram(int index, bool loadImageData) const if (globalTable.containsLabel(EMDL_TOMO_TILT_SERIES_NAME)) { // option A: Kino's original IMOD import functionality - globalTable.getValueSafely(EMDL_TOMO_TILT_SERIES_NAME, out.tiltSeriesFilename, index); globalTable.getValueSafely(EMDL_TOMO_FRAME_COUNT, out.frameCount, index); @@ -299,7 +278,7 @@ Tomogram TomogramSet::loadTomogram(int index, bool loadImageData) const EMDL_TOMO_PROJECTION_Z, EMDL_TOMO_PROJECTION_W }); - for (int i = 0; i < 4; i++) + for (int i = 0; i < 4; i++) { std::vector vals; m.getValueSafely(rows[i], vals, f); @@ -669,10 +648,10 @@ void TomogramSet::convertBackFromSingleMetaDataTable(MetaDataTable &MDin) MetaDataTable MDsub = subsetMetaDataTable(MDin, EMDL_IMAGE_OPTICS_GROUP, t+1, t+1); // Make sure no one unsorted the tilt images in each serie... - if (MDsub.containsLabel(EMDL_MICROGRAPH_PRE_EXPOSURE)) + if (MDsub.containsLabel(EMDL_TOMO_TILT_MOVIE_INDEX)) MDsub.newSort(EMDL_MICROGRAPH_PRE_EXPOSURE); else - REPORT_ERROR("BUG: MDsub does no longer contain a rlnMicrographPreExposure label"); + REPORT_ERROR("BUG: MDsub does no longer contain a rlnTomoTiltMovieIndex label"); if (MDsub.numberOfObjects() != tomogramTables[t].numberOfObjects()) { From e8b5ffdfbb66e1f1316f73723dfaf7ad4cecbf25 Mon Sep 17 00:00:00 2001 From: scheres Date: Tue, 12 Jul 2022 17:53:44 +0100 Subject: [PATCH 089/495] better check of combination of is_tomo with fn_tomo being empty --- src/exp_model.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/exp_model.cpp b/src/exp_model.cpp index ba6b01a49..ae3612031 100644 --- a/src/exp_model.cpp +++ b/src/exp_model.cpp @@ -831,6 +831,8 @@ void Experiment::read(FileName fn_exp, FileName fn_tomo, FileName fn_motion, MDgen.read(fn_exp, "general"); if (MDgen.numberOfObjects() > 0) MDgen.getValue(EMDL_TOMO_SUBTOMOGRAM_STACK2D, is_tomo); + else + is_tomo = false; // MDimg and MDopt have to be read at the same time, so that the optics groups can be // renamed in case they are non-contiguous or not sorted @@ -838,14 +840,16 @@ void Experiment::read(FileName fn_exp, FileName fn_tomo, FileName fn_motion, nr_images_per_optics_group.resize(obsModel.numberOfOpticsGroups(), 0); std::vector> particles_idx; - if (fn_tomo != "") + if (is_tomo) { + + if (fn_tomo == "") REPORT_ERROR("ERROR: you need to provide fn_tomo when refining 2D stacks of tilt series images"); + // For now read in particle table twice: once into MDimg and once into particleSet... // TODO: work with pointer to avoid duplication? // TODO: ObservationModel::loadSafely(fn_exp, obsModel, MDimg, "particles", verb, true, true); tomogramSet.read(fn_tomo, verb>0); particleSet.read(fn_exp, fn_motion, verb>0); - is_tomo = true; // For checking tomogram sanity below particles_idx = particleSet.splitByTomogram(tomogramSet, verb>0); From 311dcb0e8f0f89d381dac7fdbc59d0b907bf2699 Mon Sep 17 00:00:00 2001 From: alisterburt Date: Wed, 13 Jul 2022 11:27:15 +0100 Subject: [PATCH 090/495] better gui label for dose in tomo import --- src/pipeline_jobs.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipeline_jobs.cpp b/src/pipeline_jobs.cpp index 98d16b47d..c3f016784 100644 --- a/src/pipeline_jobs.cpp +++ b/src/pipeline_jobs.cpp @@ -6069,7 +6069,7 @@ void RelionJob::initialiseTomoImportJob() joboptions["kV"] = JobOption("Voltage (kV):", 300, 80, 300, 10, "Voltage the microscope was operated on (in kV; default=300)."); joboptions["Cs"] = JobOption("Spherical aberration (mm):", 2.7, 0.01, 4, 0.1 , "Spherical aberration of the microscope used to collect these images (in mm). Typical values are 2.7 (FEI Titan & Talos, most JEOL CRYO-ARM), 2.0 (FEI Polara), 1.4 (some JEOL CRYO-ARM) and 0.01 (microscopes with a Cs corrector)."); joboptions["Q0"] = JobOption("Amplitude contrast:", 0.1, 0.05, 1, 0.01, "Fraction of amplitude contrast (default=0.1). Often values around 10% work better than theoretically more accurate lower values. "); - joboptions["dose"] = JobOption("Frame dose:", 3.0, 0.0, 10.0, 0.1 , "Electron dose (in e/A^2) per frame (image) in the tilt series."); + joboptions["dose"] = JobOption("Dose per tilt-image:", 3.0, 0.0, 10.0, 0.1 , "Electron dose (in e/A^2) per image in the tilt series."); joboptions["do_tiltseries"]= JobOption("Import tilt-series?", true, "Set this to Yes for importing tilt movies from SerialEM mdoc format metadata."); joboptions["movie_files"] = JobOption("Tilt image movie files:", (std::string)"frames/*.mrc","File pattern pointing to the raw movie files for the tilt images"); From 84ea2bce8ce26e259eec30b7ff624e4fb6ee930d Mon Sep 17 00:00:00 2001 From: scheres Date: Mon, 18 Jul 2022 09:42:25 +0100 Subject: [PATCH 091/495] repaired bug in erquiring EMDL_TOMO_TILT_SERIES_PIXEL_SIZE in tomogram_set --- src/jaz/tomography/tomogram_set.cpp | 13 +++++++++---- src/motioncorr_runner.cpp | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/jaz/tomography/tomogram_set.cpp b/src/jaz/tomography/tomogram_set.cpp index 4545507b2..56ce2ff14 100644 --- a/src/jaz/tomography/tomogram_set.cpp +++ b/src/jaz/tomography/tomogram_set.cpp @@ -254,7 +254,12 @@ Tomogram TomogramSet::loadTomogram(int index, bool loadImageData) const double Q0; - globalTable.getValueSafely(EMDL_TOMO_TILT_SERIES_PIXEL_SIZE, out.optics.pixelSize, index); + // Now that we do notioncorrection on tomogram_sets, the micrograph pixel size may not have been set yet... + if (globalTable.containsLabel(EMDL_TOMO_TILT_SERIES_PIXEL_SIZE)) + globalTable.getValueSafely(EMDL_TOMO_TILT_SERIES_PIXEL_SIZE, out.optics.pixelSize, index); + else + out.optics.pixelSize = -999.; + globalTable.getValueSafely(EMDL_CTF_VOLTAGE, out.optics.voltage, index); globalTable.getValueSafely(EMDL_CTF_CS, out.optics.Cs, index); globalTable.getValueSafely(EMDL_CTF_Q0, Q0, index); @@ -631,16 +636,16 @@ void TomogramSet::generateSingleMetaDataTable(MetaDataTable &MDout, ObservationM for (long int t = 0; t < tomogramTables.size(); t++) { // Store all the necessary optics stuff in an opticsGroup per tomogram - RFLOAT pixelSize, voltage, Cs, Q0; + RFLOAT moviePixelSize, voltage, Cs, Q0; std::string tomo_name = getTomogramName(t); - globalTable.getValueSafely(EMDL_TOMO_TILT_SERIES_PIXEL_SIZE, pixelSize, t); + globalTable.getValueSafely(EMDL_MICROGRAPH_ORIGINAL_PIXEL_SIZE, moviePixelSize, t); globalTable.getValueSafely(EMDL_CTF_VOLTAGE, voltage, t); globalTable.getValueSafely(EMDL_CTF_CS, Cs, t); globalTable.getValueSafely(EMDL_CTF_Q0, Q0, t); obsModel.opticsMdt.addObject(); obsModel.opticsMdt.setValue(EMDL_IMAGE_OPTICS_GROUP_NAME, tomo_name); obsModel.opticsMdt.setValue(EMDL_IMAGE_OPTICS_GROUP, t+1); - obsModel.opticsMdt.setValue(EMDL_TOMO_TILT_SERIES_PIXEL_SIZE, pixelSize); + obsModel.opticsMdt.setValue(EMDL_MICROGRAPH_ORIGINAL_PIXEL_SIZE, moviePixelSize); obsModel.opticsMdt.setValue(EMDL_CTF_VOLTAGE, voltage); obsModel.opticsMdt.setValue(EMDL_CTF_CS, Cs); obsModel.opticsMdt.setValue(EMDL_CTF_Q0, Q0); diff --git a/src/motioncorr_runner.cpp b/src/motioncorr_runner.cpp index d0f7a4906..0a2a667a6 100644 --- a/src/motioncorr_runner.cpp +++ b/src/motioncorr_runner.cpp @@ -931,7 +931,7 @@ void MotioncorrRunner::generateLogFilePDFAndWriteStarFiles() if (verb > 0) progress_bar(fn_ori_micrographs.size()); // Write out STAR files at the end - // In the opticsMdt, set EMDL_MICROGRAPH_PIXEL_SIZE (i.e. possibly binned pixel size). + // In the opticsMdt, set EMDL_MICROGRAPH_PIXEL_SIZE (i.e. possibly binned pixel size) for SPA and EMDL_TOMO_TILT_SERIES_PIXEL_SIZE for STA // Keep EMDL_MICROGRAPH_ORIGINAL_PIXEL_SIZE for MTF correction if (is_tomo) { From 33889b307558ed2a3f0a187cf09de1e19fe8a58f Mon Sep 17 00:00:00 2001 From: scheres Date: Mon, 18 Jul 2022 11:32:20 +0100 Subject: [PATCH 092/495] set weights name to empty for 2D stacks --- src/jaz/tomography/programs/subtomo.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jaz/tomography/programs/subtomo.cpp b/src/jaz/tomography/programs/subtomo.cpp index aadc08e44..741a269da 100644 --- a/src/jaz/tomography/programs/subtomo.cpp +++ b/src/jaz/tomography/programs/subtomo.cpp @@ -487,7 +487,7 @@ void SubtomoProgram::processTomograms( part_id, t, particleSet, tomogramSet); std::string outData = (do_stack2d) ? filenameRoot + "_stack2d.mrcs" : filenameRoot + "_data.mrc"; - std::string outWeight = filenameRoot + "_weights.mrc"; + std::string outWeight = (do_stack2d) ? "" : filenameRoot + "_weights.mrc"; std::string outCTF = filenameRoot + "_CTF2.mrc"; std::string outDiv = filenameRoot + "_div.mrc"; std::string outMulti = filenameRoot + "_multi.mrc"; From 10fcdad80823011e140a0571bd7776da286b3f70 Mon Sep 17 00:00:00 2001 From: scheres Date: Mon, 18 Jul 2022 11:47:43 +0100 Subject: [PATCH 093/495] repaired bu in passing images for 2dstacks to getFTandCTFs --- src/ml_optimiser.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/ml_optimiser.cpp b/src/ml_optimiser.cpp index 05b6c9397..cf968b018 100644 --- a/src/ml_optimiser.cpp +++ b/src/ml_optimiser.cpp @@ -5829,7 +5829,7 @@ void MlOptimiser::getFourierTransformsAndCtfs( img().resize(image_full_size[optics_group], image_full_size[optics_group]); FOR_ALL_DIRECT_ELEMENTS_IN_ARRAY2D(img()) { - DIRECT_A2D_ELEM(img(), i, j) = DIRECT_A3D_ELEM(exp_imagedata, imagedata_offset, i, j); + DIRECT_A2D_ELEM(img(), i, j) = DIRECT_A3D_ELEM(exp_imagedata, imagedata_offset + img_id, i, j); } img().setXmippOrigin(); if (has_converged && do_use_reconstruct_images) @@ -5837,11 +5837,13 @@ void MlOptimiser::getFourierTransformsAndCtfs( /// TODO: this will be WRONG for multi-image particles, but I guess that's not going to happen anyway... int my_nr_particles = exp_my_last_part_id - exp_my_first_part_id + 1; + if (mydata.is_tomo) REPORT_ERROR("ERROR: you can not use reconstruct images for 2Dstack-subtomograms!"); + ////////////// TODO: think this through for no-threads here..... rec_img().resize(image_full_size[optics_group], image_full_size[optics_group]); FOR_ALL_DIRECT_ELEMENTS_IN_ARRAY2D(rec_img()) { - DIRECT_A2D_ELEM(rec_img(), i, j) = DIRECT_A3D_ELEM(exp_imagedata, my_nr_particles + imagedata_offset, i, j); + DIRECT_A2D_ELEM(rec_img(), i, j) = DIRECT_A3D_ELEM(exp_imagedata, my_nr_particles + imagedata_offset + img_id, i, j); } rec_img().setXmippOrigin(); } From eff67578a8f65256ba625bdab09d61d3096fa876 Mon Sep 17 00:00:00 2001 From: scheres Date: Mon, 18 Jul 2022 13:02:57 +0100 Subject: [PATCH 094/495] also debug acc version for img_id indexing of exp_imgs --- src/acc/acc_ml_optimiser_impl.h | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/acc/acc_ml_optimiser_impl.h b/src/acc/acc_ml_optimiser_impl.h index 906898127..cb8b4e33d 100644 --- a/src/acc/acc_ml_optimiser_impl.h +++ b/src/acc/acc_ml_optimiser_impl.h @@ -289,7 +289,7 @@ void getFourierTransformsAndCtfs(long int part_id, else { CTIC(accMLO->timer,"ParaRead2DImages"); - img() = baseMLO->exp_imgs[op.imagedata_offset]; + img() = baseMLO->exp_imgs[op.imagedata_offset + img_id]; CTOC(accMLO->timer,"ParaRead2DImages"); } } @@ -341,7 +341,7 @@ void getFourierTransformsAndCtfs(long int part_id, img().resize(baseMLO->image_full_size[optics_group], baseMLO->image_full_size[optics_group]); FOR_ALL_DIRECT_ELEMENTS_IN_ARRAY2D(img()) { - DIRECT_A2D_ELEM(img(), i, j) = DIRECT_A3D_ELEM(baseMLO->exp_imagedata, op.imagedata_offset, i, j); + DIRECT_A2D_ELEM(img(), i, j) = DIRECT_A3D_ELEM(baseMLO->exp_imagedata, op.imagedata_offset + img_id, i, j); } img().setXmippOrigin(); if (baseMLO->has_converged && baseMLO->do_use_reconstruct_images) @@ -349,10 +349,12 @@ void getFourierTransformsAndCtfs(long int part_id, /// TODO: this will be WRONG for multi-image particles, but I guess that's not going to happen anyway... int my_nr_particles = baseMLO->exp_my_last_part_id - baseMLO->exp_my_first_part_id + 1; + if (baseMLO->mydata.is_tomo) REPORT_ERROR("ERROR: you can not use reconstruct images for 2Dstack-subtomograms!"); + rec_img().resize(baseMLO->image_full_size[optics_group], baseMLO->image_full_size[optics_group]); FOR_ALL_DIRECT_ELEMENTS_IN_ARRAY2D(rec_img()) { - DIRECT_A2D_ELEM(rec_img(), i, j) = DIRECT_A3D_ELEM(baseMLO->exp_imagedata, my_nr_particles + op.imagedata_offset, i, j); + DIRECT_A2D_ELEM(rec_img(), i, j) = DIRECT_A3D_ELEM(baseMLO->exp_imagedata, my_nr_particles + op.imagedata_offset + img_id, i, j); } rec_img().setXmippOrigin(); } From a61cceaf4d01c6a4aea78108956b972a18c50295 Mon Sep 17 00:00:00 2001 From: scheres Date: Mon, 18 Jul 2022 14:08:34 +0100 Subject: [PATCH 095/495] repaired bug in skip_align indexing of euler angles and put back in +img_id in exp_imgs --- src/ml_optimiser.cpp | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/ml_optimiser.cpp b/src/ml_optimiser.cpp index cf968b018..a92bfe11e 100644 --- a/src/ml_optimiser.cpp +++ b/src/ml_optimiser.cpp @@ -3771,7 +3771,7 @@ void MlOptimiser::expectationSomeParticles(long int my_first_part_id, long int m long int my_imagedata_offset = 0; exp_imgs.clear(); int metadata_offset = 0; - for (long int part_id_sorted = my_first_part_id; part_id_sorted <= my_last_part_id; part_id_sorted++) + for (long int part_id_sorted = my_first_part_id; part_id_sorted <= my_last_part_id; part_id_sorted++, metadata_offset++) { long int part_id = mydata.sorted_idx[part_id_sorted]; @@ -3786,7 +3786,7 @@ void MlOptimiser::expectationSomeParticles(long int my_first_part_id, long int m old_rot = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ROT); old_tilt = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_TILT); old_psi = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PSI); - sampling.addOneOrientation(old_rot, old_tilt, old_psi, do_clear); + sampling.addOneOrientation(old_rot, old_tilt, old_psi, do_clear); } else if (do_only_sample_tilt) { @@ -3828,8 +3828,6 @@ void MlOptimiser::expectationSomeParticles(long int my_first_part_id, long int m do_clear, (do_helical_refine) && (!ignore_helical_symmetry), rot_deg, tilt_deg, psi_deg); // clear for first particle } - // Store total number of images in this bunch of SomeParticles - metadata_offset += mydata.numberOfImagesInParticle(part_id); // Sjors 7 March 2016 to prevent too high disk access... Read in all pooled images simultaneously // Don't do this for sub-tomograms to save RAM! @@ -5598,7 +5596,6 @@ void MlOptimiser::getFourierTransformsAndCtfs( RFLOAT rot_deg = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ROT); RFLOAT tilt_deg = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_TILT); RFLOAT psi_deg = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_PSI); - //std::cerr << " rot_deg= " << rot_deg << " tilt_deg= " << tilt_deg << " psi_deg= " << psi_deg << std::endl; if ( (do_helical_refine) && (!ignore_helical_symmetry) ) { // Calculate my_old_offset_helix_coords from my_old_offset and psi angle @@ -5780,7 +5777,7 @@ void MlOptimiser::getFourierTransformsAndCtfs( } else { - img() = exp_imgs[imagedata_offset]; + img() = exp_imgs[imagedata_offset + img_id]; } #endif } @@ -5838,7 +5835,7 @@ void MlOptimiser::getFourierTransformsAndCtfs( /// TODO: this will be WRONG for multi-image particles, but I guess that's not going to happen anyway... int my_nr_particles = exp_my_last_part_id - exp_my_first_part_id + 1; if (mydata.is_tomo) REPORT_ERROR("ERROR: you can not use reconstruct images for 2Dstack-subtomograms!"); - + ////////////// TODO: think this through for no-threads here..... rec_img().resize(image_full_size[optics_group], image_full_size[optics_group]); FOR_ALL_DIRECT_ELEMENTS_IN_ARRAY2D(rec_img()) From 93f225c641527d9b9b2bc062c099b28fdbff74f6 Mon Sep 17 00:00:00 2001 From: scheres Date: Mon, 18 Jul 2022 14:40:18 +0100 Subject: [PATCH 096/495] subtomo shifts were in the wrong direction... --- src/ml_optimiser.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ml_optimiser.cpp b/src/ml_optimiser.cpp index a92bfe11e..2f73b3c3f 100644 --- a/src/ml_optimiser.cpp +++ b/src/ml_optimiser.cpp @@ -2457,9 +2457,7 @@ void MlOptimiser::calculateSumOfPowerSpectraAndAverageImage(MultidimArray &Fimg, Multidim void MlOptimiser::selfTranslateSubtomoStack2D(MultidimArray &img, const Matrix1D &v, long int part_id, int img_id) { + + RFLOAT xshift, yshift, zshift; mydata.getTranslationInTiltSeries(part_id, img_id, XX(v), YY(v), ZZ(v), xshift, yshift, zshift); @@ -10287,7 +10287,7 @@ void MlOptimiser::selfTranslateSubtomoStack2D(MultidimArray &img, const MultidimArray FT, Faux; transformer.FourierTransform(img, FT, true); Faux = FT; - shiftImageInFourierTransform(Faux, FT, XSIZE(img), (RFLOAT)mymodel.ori_size, xshift, yshift); + shiftImageInFourierTransform(Faux, FT, XSIZE(img), (RFLOAT)mymodel.ori_size, -xshift, -yshift); transformer.inverseFourierTransform(FT, img); } \ No newline at end of file From 1fa9b417fde9cda8a923db30d2075bd71272d923 Mon Sep 17 00:00:00 2001 From: scheres Date: Mon, 18 Jul 2022 15:17:45 +0100 Subject: [PATCH 097/495] removed spurious variable --- src/exp_model.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/exp_model.cpp b/src/exp_model.cpp index ae3612031..9f37eccd0 100644 --- a/src/exp_model.cpp +++ b/src/exp_model.cpp @@ -97,7 +97,6 @@ void Experiment::getTranslationInTiltSeries(long int part_id, int img_id, double shift3d_x, double shift3d_y, double shift3d_z, double &shift2d_x, double &shift2d_y, double &shift2d_z) { - Matrix1D shift2d(3); Matrix2D Aproj = particles[part_id].images[img_id].Aproj; shift2d_x = Aproj(0,0) * shift3d_x + Aproj(0,1) * shift3d_y + Aproj(0,2) * shift3d_z; shift2d_y = Aproj(1,0) * shift3d_x + Aproj(1,1) * shift3d_y + Aproj(2,2) * shift3d_z; From ea70aae4023c156967f20af012399e7012576d81 Mon Sep 17 00:00:00 2001 From: scheres Date: Mon, 18 Jul 2022 15:20:31 +0100 Subject: [PATCH 098/495] repaired nasty typo bug in getTranslationInTiltSeries --- src/exp_model.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/exp_model.cpp b/src/exp_model.cpp index 9f37eccd0..a02477cae 100644 --- a/src/exp_model.cpp +++ b/src/exp_model.cpp @@ -99,7 +99,7 @@ void Experiment::getTranslationInTiltSeries(long int part_id, int img_id, { Matrix2D Aproj = particles[part_id].images[img_id].Aproj; shift2d_x = Aproj(0,0) * shift3d_x + Aproj(0,1) * shift3d_y + Aproj(0,2) * shift3d_z; - shift2d_y = Aproj(1,0) * shift3d_x + Aproj(1,1) * shift3d_y + Aproj(2,2) * shift3d_z; + shift2d_y = Aproj(1,0) * shift3d_x + Aproj(1,1) * shift3d_y + Aproj(1,2) * shift3d_z; shift2d_z = 0.; } From f6b49cdfb0daac15ecf2782465d740f7d81f0d37 Mon Sep 17 00:00:00 2001 From: scheres Date: Mon, 18 Jul 2022 15:53:28 +0100 Subject: [PATCH 099/495] made better error handling --- src/jaz/tomography/particle_set.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jaz/tomography/particle_set.cpp b/src/jaz/tomography/particle_set.cpp index a173dd4dd..b205a94c0 100644 --- a/src/jaz/tomography/particle_set.cpp +++ b/src/jaz/tomography/particle_set.cpp @@ -516,7 +516,7 @@ int ParticleSet::getOpticsGroup(ParticleIndex particle_id) const { if (!partTable.containsLabel(EMDL_IMAGE_OPTICS_GROUP)) { - REPORT_ERROR("ParticleSet::getPixelSize: pixel size (rlnImagePixelSize) missing from optics table"); + REPORT_ERROR("ParticleSet::getOpticsGroup: optics group (rlnOpticsGroup) is missing from optics table"); } int out; From 58e7ed676da21b56c265a9b587cbc8c5cee1c7ec Mon Sep 17 00:00:00 2001 From: scheres Date: Mon, 18 Jul 2022 15:58:22 +0100 Subject: [PATCH 100/495] removed spurious debugging statement# --- src/jaz/tomography/tomogram_set.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/jaz/tomography/tomogram_set.cpp b/src/jaz/tomography/tomogram_set.cpp index b9f4e9cee..a554e45ca 100644 --- a/src/jaz/tomography/tomogram_set.cpp +++ b/src/jaz/tomography/tomogram_set.cpp @@ -77,7 +77,6 @@ bool TomogramSet::read(FileName filename, bool verbose) { FileName name = globalTable.getString(EMDL_TOMO_NAME, t); std::string fn_star = globalTable.getString(EMDL_TOMO_TILT_SERIES_STARFILE, t); - std::cerr << " fn_star= " << fn_star << std::endl; tomogramTables[t].read(fn_star, name); if (!tomogramTables[t].containsLabel(EMDL_MICROGRAPH_PRE_EXPOSURE)) From 77d6f843c91929a0e8286b395fd4c6835723f34a Mon Sep 17 00:00:00 2001 From: scheres Date: Mon, 18 Jul 2022 17:27:32 +0100 Subject: [PATCH 101/495] repaired bugs no-round translations skip-align stack2d and big one in selfTranslateSubtomoStack2D; also switched off caklculation of angular error for now --- src/ml_optimiser.cpp | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/src/ml_optimiser.cpp b/src/ml_optimiser.cpp index 2f73b3c3f..7a6aebd71 100644 --- a/src/ml_optimiser.cpp +++ b/src/ml_optimiser.cpp @@ -3216,7 +3216,8 @@ void MlOptimiser::expectation() int n_trials_acc = (mymodel.ref_dim==3 && mymodel.data_dim != 3) ? 100 : 10; n_trials_acc = XMIPP_MIN(n_trials_acc, mydata.numberOfParticles()); getMetaAndImageDataSubset(0, n_trials_acc-1, false); - calculateExpectedAngularErrors(0, n_trials_acc-1); + std::cerr << " SKIPPING calculateExpectedAngularErrors!!! REACTIVATE AFTER DEBUGGING!!" << std::endl; + //calculateExpectedAngularErrors(0, n_trials_acc-1); } // D. Update the angular sampling (all nodes except leader) @@ -3802,13 +3803,23 @@ void MlOptimiser::expectationSomeParticles(long int my_first_part_id, long int m RFLOAT rot_deg, tilt_deg, psi_deg; my_old_offset_x = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_XOFF); my_old_offset_y = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_YOFF); - rounded_offset_x = my_old_offset_x - ROUND(my_old_offset_x); - rounded_offset_y = my_old_offset_y - ROUND(my_old_offset_y); - if (mymodel.data_dim == 3 || mydata.is_tomo) - { - my_old_offset_z = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ZOFF); - rounded_offset_z = my_old_offset_z - ROUND(my_old_offset_z); - } + + // For for 2Dstacks in STA, there is no ROUNDING!!! + if (mydata.is_tomo) + { + rounded_offset_x = rounded_offset_y = rounded_offset_z = 0.; + } + else + { + rounded_offset_x = my_old_offset_x - ROUND(my_old_offset_x); + rounded_offset_y = my_old_offset_y - ROUND(my_old_offset_y); + if (mymodel.data_dim == 3 || mydata.is_tomo) + { + my_old_offset_z = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ZOFF); + rounded_offset_z = my_old_offset_z - ROUND(my_old_offset_z); + } + } + if (do_helical_refine) { rot_deg = DIRECT_A2D_ELEM(exp_metadata, metadata_offset, METADATA_ROT); @@ -8374,6 +8385,7 @@ void MlOptimiser::storeWeightedSums(long int part_id, int ibody, { ZZ(shifts) = ZZ(exp_old_offset) + oversampled_translations_z[iover_trans]; } + #ifdef DEBUG_BODIES2 std::cerr << ihidden_over << " weight= " << weight; std::cerr << " exp_old_offset= " << exp_old_offset[img_id].transpose() << std::endl; @@ -10287,7 +10299,7 @@ void MlOptimiser::selfTranslateSubtomoStack2D(MultidimArray &img, const MultidimArray FT, Faux; transformer.FourierTransform(img, FT, true); Faux = FT; - shiftImageInFourierTransform(Faux, FT, XSIZE(img), (RFLOAT)mymodel.ori_size, -xshift, -yshift); + shiftImageInFourierTransform(Faux, FT, (RFLOAT)mymodel.ori_size, xshift, yshift); transformer.inverseFourierTransform(FT, img); } \ No newline at end of file From c504745da442161019a46d44cf6bc15bbbdadbdf Mon Sep 17 00:00:00 2001 From: scheres Date: Tue, 19 Jul 2022 15:11:59 +0100 Subject: [PATCH 102/495] reduce minimum sigma2 in BP --- src/backprojector.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backprojector.cpp b/src/backprojector.cpp index 212ba0ad5..907aad360 100644 --- a/src/backprojector.cpp +++ b/src/backprojector.cpp @@ -1078,7 +1078,7 @@ void BackProjector::updateSSNRarrays(RFLOAT tau2_fudge, // Average (inverse of) sigma2 in reconstruction FOR_ALL_DIRECT_ELEMENTS_IN_ARRAY1D(sigma2) { - if (DIRECT_A1D_ELEM(sigma2, i) > 1e-10) + if (DIRECT_A1D_ELEM(sigma2, i) > 1e-20) DIRECT_A1D_ELEM(sigma2, i) = DIRECT_A1D_ELEM(counter, i) / DIRECT_A1D_ELEM(sigma2, i); else if (DIRECT_A1D_ELEM(sigma2, i) == 0) DIRECT_A1D_ELEM(sigma2, i) = 0.; From 431c53023544cd190c8d8580b1f47961e9d128ed Mon Sep 17 00:00:00 2001 From: scheres Date: Tue, 19 Jul 2022 15:12:43 +0100 Subject: [PATCH 103/495] normalize zero-mean siddev-one in 2D stacks --- src/jaz/tomography/programs/subtomo.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/jaz/tomography/programs/subtomo.cpp b/src/jaz/tomography/programs/subtomo.cpp index 741a269da..ddd3260af 100644 --- a/src/jaz/tomography/programs/subtomo.cpp +++ b/src/jaz/tomography/programs/subtomo.cpp @@ -520,7 +520,15 @@ void SubtomoProgram::processTomograms( { BufferedImage particlesRS = NewStackHelper::inverseFourierTransformStack(particleStack); const float sign = flip_value ? -1.f : 1.f; - particlesRS *= sign; + + // Normalise to zero-mean and stddev one; also apply sign (contrast flip) + Image Itmp; + particlesRS.copyTo(Itmp); + double avg,stddev; + Itmp().computeAvgStddev(avg, stddev); + particlesRS -= avg; + particlesRS /= sign * stddev; + particlesRS.write(outData, binnedPixelSize, write_float16); } else From d2889e4e21fae9e89ddea42467c027990830e2a2 Mon Sep 17 00:00:00 2001 From: scheres Date: Tue, 19 Jul 2022 15:14:21 +0100 Subject: [PATCH 104/495] tabsincos repaired, calculateAngularError temporarily disabled --- src/ml_optimiser.cpp | 5 +++-- src/ml_optimiser_mpi.cpp | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/ml_optimiser.cpp b/src/ml_optimiser.cpp index 7a6aebd71..cbc7b01a2 100644 --- a/src/ml_optimiser.cpp +++ b/src/ml_optimiser.cpp @@ -2150,7 +2150,7 @@ void MlOptimiser::initialiseGeneral(int rank) } // Tabulated sine and cosine values (for 2D helical segments / 3D helical sub-tomogram averaging with on-the-fly shifts) - if ( (do_shifts_onthefly) && (do_helical_refine) && (!ignore_helical_symmetry) ) + if ( mydata.is_tomo || ((do_shifts_onthefly) && (do_helical_refine) && (!ignore_helical_symmetry)) ) { tab_sin.initialise(100000); tab_cos.initialise(100000); @@ -10299,7 +10299,8 @@ void MlOptimiser::selfTranslateSubtomoStack2D(MultidimArray &img, const MultidimArray FT, Faux; transformer.FourierTransform(img, FT, true); Faux = FT; - shiftImageInFourierTransform(Faux, FT, (RFLOAT)mymodel.ori_size, xshift, yshift); + + shiftImageInFourierTransformWithTabSincos(Faux, FT, (RFLOAT)mymodel.ori_size, mymodel.ori_size, tab_sin, tab_cos, xshift, yshift); transformer.inverseFourierTransform(FT, img); } \ No newline at end of file diff --git a/src/ml_optimiser_mpi.cpp b/src/ml_optimiser_mpi.cpp index 74274e8f9..6a41616b1 100644 --- a/src/ml_optimiser_mpi.cpp +++ b/src/ml_optimiser_mpi.cpp @@ -813,7 +813,8 @@ void MlOptimiserMpi::expectation() // During gradient refinement only do this every 10 iterations if (!((iter==1 && do_firstiter_cc) || do_always_cc) && !(do_skip_align && do_skip_rotate || do_only_sample_tilt) && (do_auto_refine || !do_grad || iter % 10 == 0 || iter == nr_iter || iter <= 1)) - calculateExpectedAngularErrors(0, n_trials_acc - 1); + std::cerr << " WARNING: calculateExpectedAngularErrors deactivated for debugging!" << std::endl; + //calculateExpectedAngularErrors(0, n_trials_acc - 1); #ifdef TIMING timer.toc(TIMING_EXP_2); From 8638c8cac6f69d655fdcef95d6ce7f2a67d06c23 Mon Sep 17 00:00:00 2001 From: scheres Date: Tue, 19 Jul 2022 15:21:11 +0100 Subject: [PATCH 105/495] added option for skip_wiener and move default origin to 0,0,0 --- .../programs/reconstruct_tomogram.cpp | 21 +++++++++++++------ .../programs/reconstruct_tomogram.h | 2 +- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/jaz/tomography/programs/reconstruct_tomogram.cpp b/src/jaz/tomography/programs/reconstruct_tomogram.cpp index 6194373e6..cceb21884 100644 --- a/src/jaz/tomography/programs/reconstruct_tomogram.cpp +++ b/src/jaz/tomography/programs/reconstruct_tomogram.cpp @@ -41,20 +41,29 @@ void TomoBackprojectProgram::readParameters(int argc, char *argv[]) applyWeight = !parser.checkOption("--no_weight", "Do not perform weighting in Fourier space using a Wiener filter"); applyPreWeight = parser.checkOption("--pre_weight", "Pre-weight the 2D slices prior to backprojection"); - FourierCrop = parser.checkOption("--Fc", "Downsample the 2D images by Fourier cropping"); + FourierCrop = parser.checkOption("--Fc", "Downsample the 2D images by Fourier cropping"); do_only_unfinished = parser.checkOption("--only_do_unfinished", "Only reconstruct those tomograms that haven't finished yet"); SNR = textToDouble(parser.getOption("--SNR", "SNR assumed by the Wiener filter", "10")); applyCtf = !parser.checkOption("--noctf", "Ignore the CTF"); + doWiener = !parser.checkOption("--skip_wiener", "Do multiply images with CTF, but don't divide by CTF^2 in Wiener filter"); - zeroDC = !parser.checkOption("--keep_mean", "Do not zero the DC component of each frame"); + if (!doWiener) applyCtf = true; + + zeroDC = !parser.checkOption("--keep_mean", "Do not zero the DC component of each frame"); taperDist = textToDouble(parser.getOption("--td", "Tapering distance", "0.0")); taperFalloff = textToDouble(parser.getOption("--tf", "Tapering falloff", "0.0")); - x0 = textToDouble(parser.getOption("--x0", "X origin", "1.0")); - y0 = textToDouble(parser.getOption("--y0", "Y origin", "1.0")); - z0 = textToDouble(parser.getOption("--z0", "Z origin", "1.0")); + // SHWS & Aburt 19Jul2022: use zero-origins from relion-4.1 onwards.... + //x0 = textToDouble(parser.getOption("--x0", "X origin", "1.0")); + //y0 = textToDouble(parser.getOption("--y0", "Y origin", "1.0")); + //z0 = textToDouble(parser.getOption("--z0", "Z origin", "1.0")); + + x0 = textToDouble(parser.getOption("--x0", "X origin", "0.0")); + y0 = textToDouble(parser.getOption("--y0", "Y origin", "0.0")); + z0 = textToDouble(parser.getOption("--z0", "Z origin", "0.0")); + spacing = textToDouble(parser.getOption("--bin", "Binning", "1.0")); angpix_spacing = textToDouble(parser.getOption("--binned_angpix", "OR: desired pixel size after binning", "-1")); @@ -361,7 +370,7 @@ void TomoBackprojectProgram::reconstructOneTomogram(int tomoIndex) orig, spacing, RealSpaceBackprojection::Linear, taperFalloff, taperDist); - if (applyWeight || applyCtf) + if ((applyWeight || applyCtf) && doWiener) { BufferedImage psf(w1, h1, t1); psf.fill(0.f); diff --git a/src/jaz/tomography/programs/reconstruct_tomogram.h b/src/jaz/tomography/programs/reconstruct_tomogram.h index b0be853d5..1c7cc183c 100644 --- a/src/jaz/tomography/programs/reconstruct_tomogram.h +++ b/src/jaz/tomography/programs/reconstruct_tomogram.h @@ -20,7 +20,7 @@ class TomoBackprojectProgram int w, h, d; double spacing, angpix_spacing, x0, y0, z0, taperDist, taperFalloff; FileName tomoName, outFn; - bool applyPreWeight, applyWeight, applyCtf, zeroDC, FourierCrop; + bool applyPreWeight, applyWeight, applyCtf, doWiener, zeroDC, FourierCrop; bool do_multiple, do_only_unfinished; double SNR; From bf4b52b6162b3e423e8b6d29735d2d62e9f4071d Mon Sep 17 00:00:00 2001 From: scheres Date: Tue, 19 Jul 2022 15:21:33 +0100 Subject: [PATCH 106/495] no longer have tomogram origin at 1,1,1 --- src/jaz/tomography/particle_set.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/jaz/tomography/particle_set.cpp b/src/jaz/tomography/particle_set.cpp index cbacc7641..fdcd29c6a 100644 --- a/src/jaz/tomography/particle_set.cpp +++ b/src/jaz/tomography/particle_set.cpp @@ -252,11 +252,13 @@ d3Vector ParticleSet::getPosition(ParticleIndex particle_id) const const d3Matrix A_subtomogram = getSubtomogramMatrix(particle_id); d3Vector out = getParticleCoord(particle_id) - (A_subtomogram * getParticleOffset(particle_id)) / originalPixelSize; - + + /* SHWS & ABurt 19Jul2022: let's no longer do this in relion-4.1 out.x += 1.0; out.y += 1.0; out.z += 1.0; - + */ + return out; } From fde1c944243e117f5cd87450d601767d696735ec Mon Sep 17 00:00:00 2001 From: scheres Date: Tue, 19 Jul 2022 17:13:24 +0100 Subject: [PATCH 107/495] sort on tilt series index! --- src/jaz/tomography/tomogram_set.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jaz/tomography/tomogram_set.cpp b/src/jaz/tomography/tomogram_set.cpp index a554e45ca..d16f8cf2a 100644 --- a/src/jaz/tomography/tomogram_set.cpp +++ b/src/jaz/tomography/tomogram_set.cpp @@ -653,7 +653,7 @@ void TomogramSet::convertBackFromSingleMetaDataTable(MetaDataTable &MDin) // Make sure no one unsorted the tilt images in each serie... if (MDsub.containsLabel(EMDL_TOMO_TILT_MOVIE_INDEX)) - MDsub.newSort(EMDL_MICROGRAPH_PRE_EXPOSURE); + MDsub.newSort(EMDL_TOMO_TILT_MOVIE_INDEX); else REPORT_ERROR("BUG: MDsub does no longer contain a rlnTomoTiltMovieIndex label"); From 20eba06b2a206fb8c7e584dced39245b2df30916 Mon Sep 17 00:00:00 2001 From: scheres Date: Wed, 20 Jul 2022 10:11:36 +0100 Subject: [PATCH 108/495] rewritten loops in calculateEpxectedAngularErrors to deal with multiple img_id per part_id --- src/ml_optimiser.cpp | 16535 +++++++++++++++++++------------------ src/ml_optimiser_mpi.cpp | 3 +- 2 files changed, 8270 insertions(+), 8268 deletions(-) diff --git a/src/ml_optimiser.cpp b/src/ml_optimiser.cpp index cbc7b01a2..c5ca3df19 100644 --- a/src/ml_optimiser.cpp +++ b/src/ml_optimiser.cpp @@ -24,11 +24,11 @@ //#define DEBUG_BODIES #ifdef TIMING - #define RCTIC(timer,label) (timer.tic(label)) - #define RCTOC(timer,label) (timer.toc(label)) + #define RCTIC(timer,label) (timer.tic(label)) + #define RCTOC(timer,label) (timer.toc(label)) #else - #define RCTIC(timer,label) - #define RCTOC(timer,label) + #define RCTIC(timer,label) + #define RCTOC(timer,label) #endif #include @@ -49,12 +49,12 @@ #include #endif #ifdef ALTCPU - #include - #include - #include - #define TBB_PREVIEW_GLOBAL_CONTROL 1 - #include - #include "src/acc/cpu/cpu_ml_optimiser.h" + #include + #include + #include + #define TBB_PREVIEW_GLOBAL_CONTROL 1 + #include + #include "src/acc/cpu/cpu_ml_optimiser.h" #endif #define NR_CLASS_MUTEXES 5 @@ -67,23 +67,23 @@ static omp_lock_t global_mutex; void globalThreadExpectationSomeParticles(void *self, int thread_id) { - MlOptimiser *MLO = (MlOptimiser*)self; + MlOptimiser *MLO = (MlOptimiser*)self; - try - { + try + { #ifdef _CUDA_ENABLED - if (MLO->do_gpu) - ((MlOptimiserCuda*) MLO->cudaOptimisers[thread_id])->doThreadExpectationSomeParticles(thread_id); - else -#endif - MLO->doThreadExpectationSomeParticles(thread_id); - } - catch (RelionError XE) - { - RelionError *gE = new RelionError(XE.msg, XE.file, XE.line); - gE->msg = XE.msg; - MLO->threadException = gE; - } + if (MLO->do_gpu) + ((MlOptimiserCuda*) MLO->cudaOptimisers[thread_id])->doThreadExpectationSomeParticles(thread_id); + else +#endif + MLO->doThreadExpectationSomeParticles(thread_id); + } + catch (RelionError XE) + { + RelionError *gE = new RelionError(XE.msg, XE.file, XE.line); + gE->msg = XE.msg; + MLO->threadException = gE; + } } @@ -91,440 +91,440 @@ void globalThreadExpectationSomeParticles(void *self, int thread_id) void MlOptimiser::usage() { - parser.writeUsage(std::cout); + parser.writeUsage(std::cout); } void MlOptimiser::read(int argc, char **argv, int rank) { //#define DEBUG_READ - parser.setCommandLine(argc, argv); - - if (checkParameter(argc, argv, "--continue")) - { - // Do this before reading in the data.star file below! - do_preread_images = checkParameter(argc, argv, "--preread_images"); - do_parallel_disc_io = !checkParameter(argc, argv, "--no_parallel_disc_io"); - - parser.addSection("Continue options"); - FileName fn_in = parser.getOption("--continue", "_optimiser.star file of the iteration after which to continue"); - // Read in previously calculated parameters - if (fn_in != "") - read(fn_in, rank); - - // And look for additional command-line options... - parseContinue(argc, argv); - } - else - { - // Start a new run from scratch - parseInitial(argc, argv); - } + parser.setCommandLine(argc, argv); + + if (checkParameter(argc, argv, "--continue")) + { + // Do this before reading in the data.star file below! + do_preread_images = checkParameter(argc, argv, "--preread_images"); + do_parallel_disc_io = !checkParameter(argc, argv, "--no_parallel_disc_io"); + + parser.addSection("Continue options"); + FileName fn_in = parser.getOption("--continue", "_optimiser.star file of the iteration after which to continue"); + // Read in previously calculated parameters + if (fn_in != "") + read(fn_in, rank); + + // And look for additional command-line options... + parseContinue(argc, argv); + } + else + { + // Start a new run from scratch + parseInitial(argc, argv); + } } void MlOptimiser::parseContinue(int argc, char **argv) { #ifdef DEBUG - std::cerr << "Entering parseContinue" << std::endl; -#endif - - int general_section = parser.addSection("General options"); - // Not all parameters are accessible here... - FileName fn_out_new = parser.getOption("--o", "Output rootname", "OLD_ctX"); - if (fn_out_new == "OLD_ctX" || fn_out_new == fn_out ) - fn_out += "_ct" + integerToString(iter); - else - fn_out = fn_out_new; - - do_force_converge = parser.checkOption("--force_converge", "Force an auto-refinement run to converge immediately upon continuation."); - - // For multi-body refinement - bool fn_body_masks_was_empty = (fn_body_masks == "None"); - std::string fnt; - fnt = parser.getOption("--multibody_masks", "STAR file with masks and metadata for multi-body refinement", "OLD"); - if (fnt != "OLD") - fn_body_masks = fnt; - // Don't use _ctXX at start of a multibody refinement - if (fn_body_masks_was_empty && fn_body_masks != "") - fn_out = parser.getOption("--o", "Output rootname", "run"); - - // Also allow change of padding... - fnt = parser.getOption("--pad", "Oversampling factor for the Fourier transforms of the references", "OLD"); - if (fnt != "OLD") - { - if (textToInteger(fnt) != mymodel.padding_factor) - { - if (mymodel.nr_bodies > 1) - REPORT_ERROR("ERROR: cannot change padding factor in a continuation of a multi-body refinement..."); - mymodel.padding_factor = textToInteger(fnt); - // Re-initialise the model to get the right padding factors in the PPref vectors - mymodel.initialise(); - } - } - - // Is this a new multi-body refinement? - if (fn_body_masks_was_empty && fn_body_masks != "None") - do_initialise_bodies = true; - else - do_initialise_bodies = false; - - if (do_initialise_bodies) - { - ini_high = textToFloat(parser.getOption("--ini_high", "Resolution (in Angstroms) to which to limit refinement in the first iteration ", "-1")); - - mymodel.norm_body_mask_overlap = parser.checkOption("--multibody_norm_overlap", "Overlapping regions between bodies are normalized. This reduces memory requirements."); - } - do_reconstruct_subtracted_bodies = parser.checkOption("--reconstruct_subtracted_bodies", "Use this flag to perform reconstructions with the subtracted images in multi-body refinement"); - - fnt = parser.getOption("--iter", "Maximum number of iterations to perform", "OLD"); - if (fnt != "OLD") - nr_iter = textToInteger(fnt); - - fnt = parser.getOption("--tau2_fudge", "Regularisation parameter (values higher than 1 give more weight to the data)", "OLD"); - if (fnt != "OLD") - { - mymodel.tau2_fudge_factor = textToFloat(fnt); - tau2_fudge_arg = mymodel.tau2_fudge_factor; - } - - fnt = parser.getOption("--tau2_fudge_scheme", "Tau2 fudge factor updates scheme. Valid values are plain or -step. Where is the deflate factor during initial stage.", "OLD"); - if (fnt != "OLD") - tau2_fudge_scheme = fnt; - - auto_ignore_angle_changes = parser.checkOption("--auto_ignore_angles", "In auto-refinement, update angular sampling regardless of changes in orientations for convergence. This makes convergence faster."); - auto_resolution_based_angles= parser.checkOption("--auto_resol_angles", "In auto-refinement, update angular sampling based on resolution-based required sampling. This makes convergence faster."); - allow_coarser_samplings = parser.checkOption("--allow_coarser_sampling", "In 2D/3D classification, allow coarser angular and translational samplings if accuracies are bad (typically in earlier iterations."); - - // Solvent flattening - if (parser.checkOption("--flatten_solvent", "Switch on masking on the references?", "OLD")) - do_solvent = true; - - // Check whether the mask has changed - fnt = parser.getOption("--solvent_mask", "User-provided mask for the references", "OLD"); - if (fnt != "OLD") - fn_mask = fnt; - - // Check whether the secondary mask has changed - fnt = parser.getOption("--solvent_mask2", "User-provided secondary mask", "OLD"); - if (fnt != "OLD") - fn_mask2 = fnt; - - // These are still experimental; so not in the optimiser.star yet. - fn_lowpass_mask = parser.getOption("--lowpass_mask", "User-provided mask for low-pass filtering", "None"); - lowpass = textToFloat(parser.getOption("--lowpass", "User-provided cutoff for region specified above", "0")); - - // Check whether tau2-spectrum has changed - fnt = parser.getOption("--tau", "STAR file with input tau2-spectrum (to be kept constant)", "OLD"); - if (fnt != "OLD") - fn_tau = fnt; - - // Check whether particle diameter has changed - fnt = parser.getOption("--particle_diameter", "Diameter of the circular mask that will be applied to the experimental images (in Angstroms)", "OLD"); - if (fnt != "OLD") - particle_diameter = textToFloat(fnt); - - // Gradient related - fnt = parser.getOption("--grad_em_iters", "Number of iterations at the end of a gradient refinement using Expectation-Maximization", "OLD"); - if (fnt != "OLD") - grad_em_iters = textToInteger(fnt); - - fnt = parser.getOption("--grad_stepsize", "Step size parameter for gradient optimisation.", "OLD"); - if (fnt != "OLD") - grad_stepsize = textToFloat(fnt); - - fnt = parser.getOption("--grad_stepsize_scheme", "Gradient step size updates scheme. Valid values are plain or -step. Where is the initial factor during initial stage.", "OLD"); - if (fnt != "OLD") - grad_stepsize_scheme = fnt; - - fnt = parser.getOption("--grad_ini_frac", "Fraction of iterations in the initial phase of refinement", "OLD"); - if (fnt != "OLD") - grad_ini_frac = textToFloat(fnt); - - fnt = parser.getOption("--grad_fin_frac", "Fraction of iterations in the final phase of refinement", "OLD"); - if (fnt != "OLD") - grad_fin_frac = textToFloat(fnt); - - if (grad_ini_frac <= 0 || 1 <= grad_ini_frac) - REPORT_ERROR("Invalid value for --grad_ini_frac."); - if (grad_fin_frac <= 0 || 1 <= grad_fin_frac) - REPORT_ERROR("Invalid value for --grad_fin_frac."); - - if (grad_ini_frac + grad_fin_frac > 0.9) { - RFLOAT sum = grad_ini_frac + grad_fin_frac + 0.1; - grad_ini_frac /= sum; - grad_fin_frac /= sum; - } - - grad_ini_iter = nr_iter * grad_ini_frac; - grad_fin_iter = nr_iter * grad_fin_frac; - grad_inbetween_iter = nr_iter - grad_ini_iter - grad_fin_iter; - if (grad_inbetween_iter < 0) - grad_inbetween_iter = 0; - - fnt = parser.getOption("--grad_ini_resol", "Resolution cutoff during the initial SGD iterations (A)", "OLD"); - if (fnt != "OLD") - grad_ini_resol = textToFloat(fnt); - - fnt = parser.getOption("--grad_fin_resol", "Resolution cutoff during the final SGD iterations (A)", "OLD"); - if (fnt != "OLD") - grad_fin_resol = textToFloat(fnt); - - fnt = parser.getOption("--grad_ini_subset", "Mini-batch size during the initial SGD iterations", "OLD"); - if (fnt != "OLD") - grad_ini_subset_size = textToInteger(fnt); - - fnt = parser.getOption("--grad_fin_subset", "Mini-batch size during the final SGD iterations", "OLD"); - if (fnt != "OLD") - grad_fin_subset_size = textToInteger(fnt); - - fnt = parser.getOption("--mu", "Momentum parameter for SGD updates", "OLD"); - if (fnt != "OLD") - mu = textToFloat(fnt); - - fnt = parser.getOption("--grad_write_iter", "Write out model every so many iterations in SGD", "OLD"); - if (fnt != "OLD") - write_every_grad_iter = textToInteger(fnt); + std::cerr << "Entering parseContinue" << std::endl; +#endif + + int general_section = parser.addSection("General options"); + // Not all parameters are accessible here... + FileName fn_out_new = parser.getOption("--o", "Output rootname", "OLD_ctX"); + if (fn_out_new == "OLD_ctX" || fn_out_new == fn_out ) + fn_out += "_ct" + integerToString(iter); + else + fn_out = fn_out_new; + + do_force_converge = parser.checkOption("--force_converge", "Force an auto-refinement run to converge immediately upon continuation."); + + // For multi-body refinement + bool fn_body_masks_was_empty = (fn_body_masks == "None"); + std::string fnt; + fnt = parser.getOption("--multibody_masks", "STAR file with masks and metadata for multi-body refinement", "OLD"); + if (fnt != "OLD") + fn_body_masks = fnt; + // Don't use _ctXX at start of a multibody refinement + if (fn_body_masks_was_empty && fn_body_masks != "") + fn_out = parser.getOption("--o", "Output rootname", "run"); + + // Also allow change of padding... + fnt = parser.getOption("--pad", "Oversampling factor for the Fourier transforms of the references", "OLD"); + if (fnt != "OLD") + { + if (textToInteger(fnt) != mymodel.padding_factor) + { + if (mymodel.nr_bodies > 1) + REPORT_ERROR("ERROR: cannot change padding factor in a continuation of a multi-body refinement..."); + mymodel.padding_factor = textToInteger(fnt); + // Re-initialise the model to get the right padding factors in the PPref vectors + mymodel.initialise(); + } + } + + // Is this a new multi-body refinement? + if (fn_body_masks_was_empty && fn_body_masks != "None") + do_initialise_bodies = true; + else + do_initialise_bodies = false; + + if (do_initialise_bodies) + { + ini_high = textToFloat(parser.getOption("--ini_high", "Resolution (in Angstroms) to which to limit refinement in the first iteration ", "-1")); + + mymodel.norm_body_mask_overlap = parser.checkOption("--multibody_norm_overlap", "Overlapping regions between bodies are normalized. This reduces memory requirements."); + } + do_reconstruct_subtracted_bodies = parser.checkOption("--reconstruct_subtracted_bodies", "Use this flag to perform reconstructions with the subtracted images in multi-body refinement"); + + fnt = parser.getOption("--iter", "Maximum number of iterations to perform", "OLD"); + if (fnt != "OLD") + nr_iter = textToInteger(fnt); + + fnt = parser.getOption("--tau2_fudge", "Regularisation parameter (values higher than 1 give more weight to the data)", "OLD"); + if (fnt != "OLD") + { + mymodel.tau2_fudge_factor = textToFloat(fnt); + tau2_fudge_arg = mymodel.tau2_fudge_factor; + } + + fnt = parser.getOption("--tau2_fudge_scheme", "Tau2 fudge factor updates scheme. Valid values are plain or -step. Where is the deflate factor during initial stage.", "OLD"); + if (fnt != "OLD") + tau2_fudge_scheme = fnt; + + auto_ignore_angle_changes = parser.checkOption("--auto_ignore_angles", "In auto-refinement, update angular sampling regardless of changes in orientations for convergence. This makes convergence faster."); + auto_resolution_based_angles= parser.checkOption("--auto_resol_angles", "In auto-refinement, update angular sampling based on resolution-based required sampling. This makes convergence faster."); + allow_coarser_samplings = parser.checkOption("--allow_coarser_sampling", "In 2D/3D classification, allow coarser angular and translational samplings if accuracies are bad (typically in earlier iterations."); + + // Solvent flattening + if (parser.checkOption("--flatten_solvent", "Switch on masking on the references?", "OLD")) + do_solvent = true; + + // Check whether the mask has changed + fnt = parser.getOption("--solvent_mask", "User-provided mask for the references", "OLD"); + if (fnt != "OLD") + fn_mask = fnt; + + // Check whether the secondary mask has changed + fnt = parser.getOption("--solvent_mask2", "User-provided secondary mask", "OLD"); + if (fnt != "OLD") + fn_mask2 = fnt; + + // These are still experimental; so not in the optimiser.star yet. + fn_lowpass_mask = parser.getOption("--lowpass_mask", "User-provided mask for low-pass filtering", "None"); + lowpass = textToFloat(parser.getOption("--lowpass", "User-provided cutoff for region specified above", "0")); + + // Check whether tau2-spectrum has changed + fnt = parser.getOption("--tau", "STAR file with input tau2-spectrum (to be kept constant)", "OLD"); + if (fnt != "OLD") + fn_tau = fnt; + + // Check whether particle diameter has changed + fnt = parser.getOption("--particle_diameter", "Diameter of the circular mask that will be applied to the experimental images (in Angstroms)", "OLD"); + if (fnt != "OLD") + particle_diameter = textToFloat(fnt); + + // Gradient related + fnt = parser.getOption("--grad_em_iters", "Number of iterations at the end of a gradient refinement using Expectation-Maximization", "OLD"); + if (fnt != "OLD") + grad_em_iters = textToInteger(fnt); + + fnt = parser.getOption("--grad_stepsize", "Step size parameter for gradient optimisation.", "OLD"); + if (fnt != "OLD") + grad_stepsize = textToFloat(fnt); + + fnt = parser.getOption("--grad_stepsize_scheme", "Gradient step size updates scheme. Valid values are plain or -step. Where is the initial factor during initial stage.", "OLD"); + if (fnt != "OLD") + grad_stepsize_scheme = fnt; + + fnt = parser.getOption("--grad_ini_frac", "Fraction of iterations in the initial phase of refinement", "OLD"); + if (fnt != "OLD") + grad_ini_frac = textToFloat(fnt); + + fnt = parser.getOption("--grad_fin_frac", "Fraction of iterations in the final phase of refinement", "OLD"); + if (fnt != "OLD") + grad_fin_frac = textToFloat(fnt); + + if (grad_ini_frac <= 0 || 1 <= grad_ini_frac) + REPORT_ERROR("Invalid value for --grad_ini_frac."); + if (grad_fin_frac <= 0 || 1 <= grad_fin_frac) + REPORT_ERROR("Invalid value for --grad_fin_frac."); + + if (grad_ini_frac + grad_fin_frac > 0.9) { + RFLOAT sum = grad_ini_frac + grad_fin_frac + 0.1; + grad_ini_frac /= sum; + grad_fin_frac /= sum; + } + + grad_ini_iter = nr_iter * grad_ini_frac; + grad_fin_iter = nr_iter * grad_fin_frac; + grad_inbetween_iter = nr_iter - grad_ini_iter - grad_fin_iter; + if (grad_inbetween_iter < 0) + grad_inbetween_iter = 0; + + fnt = parser.getOption("--grad_ini_resol", "Resolution cutoff during the initial SGD iterations (A)", "OLD"); + if (fnt != "OLD") + grad_ini_resol = textToFloat(fnt); + + fnt = parser.getOption("--grad_fin_resol", "Resolution cutoff during the final SGD iterations (A)", "OLD"); + if (fnt != "OLD") + grad_fin_resol = textToFloat(fnt); + + fnt = parser.getOption("--grad_ini_subset", "Mini-batch size during the initial SGD iterations", "OLD"); + if (fnt != "OLD") + grad_ini_subset_size = textToInteger(fnt); + + fnt = parser.getOption("--grad_fin_subset", "Mini-batch size during the final SGD iterations", "OLD"); + if (fnt != "OLD") + grad_fin_subset_size = textToInteger(fnt); + + fnt = parser.getOption("--mu", "Momentum parameter for SGD updates", "OLD"); + if (fnt != "OLD") + mu = textToFloat(fnt); + + fnt = parser.getOption("--grad_write_iter", "Write out model every so many iterations in SGD", "OLD"); + if (fnt != "OLD") + write_every_grad_iter = textToInteger(fnt); fnt = parser.getOption("--class_inactivity_threshold", "Replace classes with little activity during gradient based classification.", "OLD"); if (fnt != "OLD") class_inactivity_threshold = textToFloat(fnt); - fnt = parser.getOption("--maxsig", "Maximum number of poses & translations to consider", "OLD"); - if (fnt != "OLD") - maximum_significants_arg = textToInteger(fnt); - - do_join_random_halves = parser.checkOption("--join_random_halves", "Join previously split random halves again (typically to perform a final reconstruction)."); - - // ORIENTATIONS - int orientations_section = parser.addSection("Orientations"); - - fnt = parser.getOption("--oversampling", "Adaptive oversampling order to speed-up calculations (0=no oversampling, 1=2x, 2=4x, etc)", "OLD"); - if (fnt != "OLD") - adaptive_oversampling = textToInteger(fnt); - - // Check whether angular sampling has changed - // Do not do this for auto_refine, but make sure to do this when initialising multi-body refinement! - if (!(do_auto_refine || do_auto_sampling) || do_initialise_bodies) - { - directions_have_changed = false; - fnt = parser.getOption("--healpix_order", "Healpix order for the angular sampling rate on the sphere (before oversampling): hp2=15deg, hp3=7.5deg, etc", "OLD"); - if (fnt != "OLD") - { - int _order = textToInteger(fnt); - if (_order != sampling.healpix_order) - { - directions_have_changed = true; - sampling.healpix_order = _order; - } - } - - fnt = parser.getOption("--psi_step", "Angular sampling (before oversampling) for the in-plane angle (default=10deg for 2D, hp sampling for 3D)", "OLD"); - if (fnt != "OLD") - sampling.psi_step = textToFloat(fnt); - - fnt = parser.getOption("--offset_range", "Search range for origin offsets (in pixels)", "OLD"); - if (fnt != "OLD") - { - sampling.offset_range = textToFloat(fnt); - sampling.offset_range *= mymodel.pixel_size; // sampling.offset_range is in Angstroms, but command line in pixels! - } - - fnt = parser.getOption("--offset_step", "Sampling rate for origin offsets (in pixels)", "OLD"); - if (fnt != "OLD") - { - sampling.offset_step = textToFloat(fnt); - sampling.offset_step *= mymodel.pixel_size; // sampling.offset_step is in Angstroms, but command line in pixels! - } - } - - fnt = parser.getOption("--auto_local_healpix_order", "Minimum healpix order (before oversampling) from which auto-refine procedure will use local searches", "OLD"); - if (fnt != "OLD") - autosampling_hporder_local_searches = textToInteger(fnt); - - // Check whether the prior mode changes - RFLOAT _sigma_rot, _sigma_tilt, _sigma_psi, _sigma_off; - int _mode; - fnt = parser.getOption("--sigma_ang", "Stddev on all three Euler angles for local angular searches (of +/- 3 stddev)", "OLD"); - if (fnt != "OLD") - { - mymodel.orientational_prior_mode = PRIOR_ROTTILT_PSI; - mymodel.sigma2_rot = mymodel.sigma2_tilt = mymodel.sigma2_psi = textToFloat(fnt) * textToFloat(fnt); - } - fnt = parser.getOption("--sigma_rot", "Stddev on the first Euler angle for local angular searches (of +/- 3 stddev)", "OLD"); - if (fnt != "OLD") - { - mymodel.orientational_prior_mode = PRIOR_ROTTILT_PSI; - mymodel.sigma2_rot = textToFloat(fnt) * textToFloat(fnt); - } - fnt = parser.getOption("--sigma_tilt", "Stddev on the first Euler angle for local angular searches (of +/- 3 stddev)", "OLD"); - if (fnt != "OLD") - { - mymodel.orientational_prior_mode = PRIOR_ROTTILT_PSI; - mymodel.sigma2_tilt = textToFloat(fnt) * textToFloat(fnt); - } - fnt = parser.getOption("--sigma_psi", "Stddev on the in-plane angle for local angular searches (of +/- 3 stddev)", "OLD"); - if (fnt != "OLD") - { - mymodel.orientational_prior_mode = PRIOR_ROTTILT_PSI; - mymodel.sigma2_psi = textToFloat(fnt) * textToFloat(fnt); - } - fnt = parser.getOption("--sigma_off", "Stddev. on the translations", "OLD"); - if (fnt != "OLD") - { - mymodel.sigma2_offset = textToFloat(fnt) * textToFloat(fnt); - } - fnt = parser.getOption("--helical_inner_diameter", "Inner diameter of helical tubes in Angstroms (for masks of helical references and particles)", "OLD"); - if (fnt != "OLD") - { - helical_tube_inner_diameter = textToFloat(fnt); - } - fnt = parser.getOption("--helical_outer_diameter", "Outer diameter of helical tubes in Angstroms (for masks of helical references and particles)", "OLD"); - if (fnt != "OLD") - { - helical_tube_outer_diameter = textToFloat(fnt); - } - fnt = parser.getOption("--perturb", "Perturbation factor for the angular sampling (0=no perturb; 0.5=perturb)", "OLD"); - if (fnt != "OLD") - { - sampling.perturbation_factor = textToFloat(fnt); - } - - if (parser.checkOption("--skip_align", "Skip orientational assignment (only classify)?")) - do_skip_align = true; - else - do_skip_align = false; // do_skip_align should normally be false... - - if (parser.checkOption("--skip_rotate", "Skip rotational assignment (only translate and classify)?")) - do_skip_rotate = true; - else - do_skip_rotate = false; // do_skip_rotate should normally be false... - - if (parser.checkOption("--bimodal_psi", "Do bimodal searches of psi angle?")) // Oct07,2015 - Shaoda, bimodal psi - do_bimodal_psi = true; - else - do_bimodal_psi = false; - - if (parser.checkOption("--center_classes", "Re-center classes based on their center-of-mass?")) - do_center_classes = true; - else - do_center_classes = false; - - do_skip_maximization = parser.checkOption("--skip_maximize", "Skip maximization step (only write out data.star file)?"); - - int corrections_section = parser.addSection("Corrections"); - - do_ctf_padding = parser.checkOption("--pad_ctf", "Perform CTF padding to treat CTF aliaising better?"); - if (do_ctf_padding) - REPORT_ERROR("--pad_ctf currently disabled."); - - // Can also switch the following option OFF - if (parser.checkOption("--scale", "Switch on intensity-scale corrections on image groups", "OLD")) - do_scale_correction = true; - if (parser.checkOption("--no_scale", "Switch off intensity-scale corrections on image groups", "OLD")) - do_scale_correction = false; - - // Can also switch the following option OFF - if (parser.checkOption("--norm", "Switch on normalisation-error correction","OLD")) - do_norm_correction = true; - if (parser.checkOption("--no_norm", "Switch off normalisation-error correction","OLD")) - do_norm_correction = false; - - int subtomogram_section = parser.addSection("Subtomogram averaging"); - normalised_subtomos = parser.checkOption("--normalised_subtomo", "Have subtomograms been multiplicity normalised? (Default=False)"); - do_skip_subtomo_correction = parser.checkOption("--skip_subtomo_multi", "Skip subtomo multiplicity correction"); - ctf3d_squared = !parser.checkOption("--ctf3d_not_squared", "CTF3D files contain sqrt(CTF^2) patterns"); - subtomo_multi_thr = textToFloat(parser.getOption("--subtomo_multi_thr", "Threshold to remove marginal voxels during expectation", "0.01")); - - int computation_section = parser.addSection("Computation"); - - x_pool = textToInteger(parser.getOption("--pool", "Number of images to pool for each thread task", "1")); - nr_threads = textToInteger(parser.getOption("--j", "Number of threads to run in parallel (only useful on multi-core machines)", "1")); - do_parallel_disc_io = !parser.checkOption("--no_parallel_disc_io", "Do NOT let parallel (MPI) processes access the disc simultaneously (use this option with NFS)"); - combine_weights_thru_disc = !parser.checkOption("--dont_combine_weights_via_disc", "Send the large arrays of summed weights through the MPI network, instead of writing large files to disc"); - do_shifts_onthefly = parser.checkOption("--onthefly_shifts", "Calculate shifted images on-the-fly, do not store precalculated ones in memory"); - do_preread_images = parser.checkOption("--preread_images", "Use this to let the leader process read all particles into memory. Be careful you have enough RAM for large data sets!"); - fn_scratch = parser.getOption("--scratch_dir", "If provided, particle stacks will be copied to this local scratch disk prior to refinement.", ""); - keep_free_scratch_Gb = textToFloat(parser.getOption("--keep_free_scratch", "Space available for copying particle stacks (in Gb)", "10")); - do_reuse_scratch = parser.checkOption("--reuse_scratch", "Re-use data on scratchdir, instead of wiping it and re-copying all data. This works only when ALL particles have already been cached."); - keep_scratch = parser.checkOption("--keep_scratch", "Don't remove scratch after convergence. Following jobs that use EXACTLY the same particles should use --reuse_scratch."); + fnt = parser.getOption("--maxsig", "Maximum number of poses & translations to consider", "OLD"); + if (fnt != "OLD") + maximum_significants_arg = textToInteger(fnt); + + do_join_random_halves = parser.checkOption("--join_random_halves", "Join previously split random halves again (typically to perform a final reconstruction)."); + + // ORIENTATIONS + int orientations_section = parser.addSection("Orientations"); + + fnt = parser.getOption("--oversampling", "Adaptive oversampling order to speed-up calculations (0=no oversampling, 1=2x, 2=4x, etc)", "OLD"); + if (fnt != "OLD") + adaptive_oversampling = textToInteger(fnt); + + // Check whether angular sampling has changed + // Do not do this for auto_refine, but make sure to do this when initialising multi-body refinement! + if (!(do_auto_refine || do_auto_sampling) || do_initialise_bodies) + { + directions_have_changed = false; + fnt = parser.getOption("--healpix_order", "Healpix order for the angular sampling rate on the sphere (before oversampling): hp2=15deg, hp3=7.5deg, etc", "OLD"); + if (fnt != "OLD") + { + int _order = textToInteger(fnt); + if (_order != sampling.healpix_order) + { + directions_have_changed = true; + sampling.healpix_order = _order; + } + } + + fnt = parser.getOption("--psi_step", "Angular sampling (before oversampling) for the in-plane angle (default=10deg for 2D, hp sampling for 3D)", "OLD"); + if (fnt != "OLD") + sampling.psi_step = textToFloat(fnt); + + fnt = parser.getOption("--offset_range", "Search range for origin offsets (in pixels)", "OLD"); + if (fnt != "OLD") + { + sampling.offset_range = textToFloat(fnt); + sampling.offset_range *= mymodel.pixel_size; // sampling.offset_range is in Angstroms, but command line in pixels! + } + + fnt = parser.getOption("--offset_step", "Sampling rate for origin offsets (in pixels)", "OLD"); + if (fnt != "OLD") + { + sampling.offset_step = textToFloat(fnt); + sampling.offset_step *= mymodel.pixel_size; // sampling.offset_step is in Angstroms, but command line in pixels! + } + } + + fnt = parser.getOption("--auto_local_healpix_order", "Minimum healpix order (before oversampling) from which auto-refine procedure will use local searches", "OLD"); + if (fnt != "OLD") + autosampling_hporder_local_searches = textToInteger(fnt); + + // Check whether the prior mode changes + RFLOAT _sigma_rot, _sigma_tilt, _sigma_psi, _sigma_off; + int _mode; + fnt = parser.getOption("--sigma_ang", "Stddev on all three Euler angles for local angular searches (of +/- 3 stddev)", "OLD"); + if (fnt != "OLD") + { + mymodel.orientational_prior_mode = PRIOR_ROTTILT_PSI; + mymodel.sigma2_rot = mymodel.sigma2_tilt = mymodel.sigma2_psi = textToFloat(fnt) * textToFloat(fnt); + } + fnt = parser.getOption("--sigma_rot", "Stddev on the first Euler angle for local angular searches (of +/- 3 stddev)", "OLD"); + if (fnt != "OLD") + { + mymodel.orientational_prior_mode = PRIOR_ROTTILT_PSI; + mymodel.sigma2_rot = textToFloat(fnt) * textToFloat(fnt); + } + fnt = parser.getOption("--sigma_tilt", "Stddev on the first Euler angle for local angular searches (of +/- 3 stddev)", "OLD"); + if (fnt != "OLD") + { + mymodel.orientational_prior_mode = PRIOR_ROTTILT_PSI; + mymodel.sigma2_tilt = textToFloat(fnt) * textToFloat(fnt); + } + fnt = parser.getOption("--sigma_psi", "Stddev on the in-plane angle for local angular searches (of +/- 3 stddev)", "OLD"); + if (fnt != "OLD") + { + mymodel.orientational_prior_mode = PRIOR_ROTTILT_PSI; + mymodel.sigma2_psi = textToFloat(fnt) * textToFloat(fnt); + } + fnt = parser.getOption("--sigma_off", "Stddev. on the translations", "OLD"); + if (fnt != "OLD") + { + mymodel.sigma2_offset = textToFloat(fnt) * textToFloat(fnt); + } + fnt = parser.getOption("--helical_inner_diameter", "Inner diameter of helical tubes in Angstroms (for masks of helical references and particles)", "OLD"); + if (fnt != "OLD") + { + helical_tube_inner_diameter = textToFloat(fnt); + } + fnt = parser.getOption("--helical_outer_diameter", "Outer diameter of helical tubes in Angstroms (for masks of helical references and particles)", "OLD"); + if (fnt != "OLD") + { + helical_tube_outer_diameter = textToFloat(fnt); + } + fnt = parser.getOption("--perturb", "Perturbation factor for the angular sampling (0=no perturb; 0.5=perturb)", "OLD"); + if (fnt != "OLD") + { + sampling.perturbation_factor = textToFloat(fnt); + } + + if (parser.checkOption("--skip_align", "Skip orientational assignment (only classify)?")) + do_skip_align = true; + else + do_skip_align = false; // do_skip_align should normally be false... + + if (parser.checkOption("--skip_rotate", "Skip rotational assignment (only translate and classify)?")) + do_skip_rotate = true; + else + do_skip_rotate = false; // do_skip_rotate should normally be false... + + if (parser.checkOption("--bimodal_psi", "Do bimodal searches of psi angle?")) // Oct07,2015 - Shaoda, bimodal psi + do_bimodal_psi = true; + else + do_bimodal_psi = false; + + if (parser.checkOption("--center_classes", "Re-center classes based on their center-of-mass?")) + do_center_classes = true; + else + do_center_classes = false; + + do_skip_maximization = parser.checkOption("--skip_maximize", "Skip maximization step (only write out data.star file)?"); + + int corrections_section = parser.addSection("Corrections"); + + do_ctf_padding = parser.checkOption("--pad_ctf", "Perform CTF padding to treat CTF aliaising better?"); + if (do_ctf_padding) + REPORT_ERROR("--pad_ctf currently disabled."); + + // Can also switch the following option OFF + if (parser.checkOption("--scale", "Switch on intensity-scale corrections on image groups", "OLD")) + do_scale_correction = true; + if (parser.checkOption("--no_scale", "Switch off intensity-scale corrections on image groups", "OLD")) + do_scale_correction = false; + + // Can also switch the following option OFF + if (parser.checkOption("--norm", "Switch on normalisation-error correction","OLD")) + do_norm_correction = true; + if (parser.checkOption("--no_norm", "Switch off normalisation-error correction","OLD")) + do_norm_correction = false; + + int subtomogram_section = parser.addSection("Subtomogram averaging"); + normalised_subtomos = parser.checkOption("--normalised_subtomo", "Have subtomograms been multiplicity normalised? (Default=False)"); + do_skip_subtomo_correction = parser.checkOption("--skip_subtomo_multi", "Skip subtomo multiplicity correction"); + ctf3d_squared = !parser.checkOption("--ctf3d_not_squared", "CTF3D files contain sqrt(CTF^2) patterns"); + subtomo_multi_thr = textToFloat(parser.getOption("--subtomo_multi_thr", "Threshold to remove marginal voxels during expectation", "0.01")); + + int computation_section = parser.addSection("Computation"); + + x_pool = textToInteger(parser.getOption("--pool", "Number of images to pool for each thread task", "1")); + nr_threads = textToInteger(parser.getOption("--j", "Number of threads to run in parallel (only useful on multi-core machines)", "1")); + do_parallel_disc_io = !parser.checkOption("--no_parallel_disc_io", "Do NOT let parallel (MPI) processes access the disc simultaneously (use this option with NFS)"); + combine_weights_thru_disc = !parser.checkOption("--dont_combine_weights_via_disc", "Send the large arrays of summed weights through the MPI network, instead of writing large files to disc"); + do_shifts_onthefly = parser.checkOption("--onthefly_shifts", "Calculate shifted images on-the-fly, do not store precalculated ones in memory"); + do_preread_images = parser.checkOption("--preread_images", "Use this to let the leader process read all particles into memory. Be careful you have enough RAM for large data sets!"); + fn_scratch = parser.getOption("--scratch_dir", "If provided, particle stacks will be copied to this local scratch disk prior to refinement.", ""); + keep_free_scratch_Gb = textToFloat(parser.getOption("--keep_free_scratch", "Space available for copying particle stacks (in Gb)", "10")); + do_reuse_scratch = parser.checkOption("--reuse_scratch", "Re-use data on scratchdir, instead of wiping it and re-copying all data. This works only when ALL particles have already been cached."); + keep_scratch = parser.checkOption("--keep_scratch", "Don't remove scratch after convergence. Following jobs that use EXACTLY the same particles should use --reuse_scratch."); #ifdef ALTCPU - do_cpu = parser.checkOption("--cpu", "Use intel vectorisation implementation for CPU"); + do_cpu = parser.checkOption("--cpu", "Use intel vectorisation implementation for CPU"); #else do_cpu = false; #endif - failsafe_threshold = textToInteger(parser.getOption("--failsafe_threshold", "Maximum number of particles permitted to be drop, due to zero sum of weights, before exiting with an error (GPU only).", "40")); + failsafe_threshold = textToInteger(parser.getOption("--failsafe_threshold", "Maximum number of particles permitted to be drop, due to zero sum of weights, before exiting with an error (GPU only).", "40")); - do_gpu = parser.checkOption("--gpu", "Use available gpu resources for some calculations"); - gpu_ids = parser.getOption("--gpu", "Device ids for each MPI-thread","default"); + do_gpu = parser.checkOption("--gpu", "Use available gpu resources for some calculations"); + gpu_ids = parser.getOption("--gpu", "Device ids for each MPI-thread","default"); #ifndef _CUDA_ENABLED if(do_gpu) - { - std::cerr << "+ WARNING : Relion was compiled without CUDA of at least version 7.0 - you do NOT have support for GPUs" << std::endl; - do_gpu = false; - } -#endif - double temp_reqSize = textToDouble(parser.getOption("--free_gpu_memory", "GPU device memory (in Mb) to leave free after allocation.", "0")); - if(!do_zero_mask) - temp_reqSize += 100; - temp_reqSize *= 1000*1000; - if(temp_reqSize<0) - REPORT_ERROR("Invalid free_gpu_memory value."); - else - requested_free_gpu_memory = temp_reqSize; - - // only allow switching ON solvent_fsc, not off - if (parser.checkOption("--solvent_correct_fsc", "Correct FSC curve for the effects of the solvent mask?")) - do_phase_random_fsc = true; - verb = textToInteger(parser.getOption("--verb", "Verbosity (1=normal, 0=silent)", "1")); - - int expert_section = parser.addSection("Expert options"); - - fnt = parser.getOption("--strict_highres_exp", "Resolution limit (in Angstrom) to restrict probability calculations in the expectation step", "OLD"); - if (fnt != "OLD") - strict_highres_exp = textToFloat(fnt); - - do_trust_ref_size = parser.checkOption("--trust_ref_size", "Trust the pixel and box size of the input reference; by default the program will die if these are different from the first optics group of the data"); - - // Debugging/analysis/hidden stuff - do_map = !checkParameter(argc, argv, "--no_map"); - minres_map = textToInteger(getParameter(argc, argv, "--minres_map", "5")); - abort_at_resolution = textToFloat(parser.getOption("--abort_at_resolution", "Abort when resolution reaches beyond this value", "-1", true)); - gridding_nr_iter = textToInteger(getParameter(argc, argv, "--gridding_iter", "10")); - debug1 = textToFloat(getParameter(argc, argv, "--debug1", "0.")); - debug2 = textToFloat(getParameter(argc, argv, "--debug2", "0.")); - debug3 = textToFloat(getParameter(argc, argv, "--debug3", "0.")); - do_bfactor = checkParameter(argc, argv, "--bfactor"); - // Read in initial sigmaNoise spectrum - fn_sigma = getParameter(argc, argv, "--sigma",""); - sigma2_fudge = textToFloat(getParameter(argc, argv, "--sigma2_fudge", "1.")); - do_acc_currentsize_despite_highres_exp = checkParameter(argc, argv, "--accuracy_current_size"); - do_sequential_halves_recons = checkParameter(argc, argv, "--sequential_halves_recons"); - do_always_join_random_halves = checkParameter(argc, argv, "--always_join_random_halves"); - do_use_all_data = checkParameter(argc, argv, "--use_all_data"); - do_always_cc = checkParameter(argc, argv, "--always_cc"); - do_only_sample_tilt = checkParameter(argc, argv, "--only_sample_tilt"); - minimum_angular_sampling = textToFloat(getParameter(argc, argv, "--minimum_angular_sampling", "0")); - maximum_angular_sampling = textToFloat(getParameter(argc, argv, "--maximum_angular_sampling", "0")); - asymmetric_padding = parser.checkOption("--asymmetric_padding", "", "false", true); - skip_gridding = !parser.checkOption("--dont_skip_gridding", "Perform gridding in the reconstruction step (obsolete?)"); - nr_iter_max = textToInteger(parser.getOption("--auto_iter_max", "In auto-refinement, stop at this iteration.", "999")); - debug_split_random_half = textToInteger(getParameter(argc, argv, "--debug_split_random_half", "0")); + { + std::cerr << "+ WARNING : Relion was compiled without CUDA of at least version 7.0 - you do NOT have support for GPUs" << std::endl; + do_gpu = false; + } +#endif + double temp_reqSize = textToDouble(parser.getOption("--free_gpu_memory", "GPU device memory (in Mb) to leave free after allocation.", "0")); + if(!do_zero_mask) + temp_reqSize += 100; + temp_reqSize *= 1000*1000; + if(temp_reqSize<0) + REPORT_ERROR("Invalid free_gpu_memory value."); + else + requested_free_gpu_memory = temp_reqSize; + + // only allow switching ON solvent_fsc, not off + if (parser.checkOption("--solvent_correct_fsc", "Correct FSC curve for the effects of the solvent mask?")) + do_phase_random_fsc = true; + verb = textToInteger(parser.getOption("--verb", "Verbosity (1=normal, 0=silent)", "1")); + + int expert_section = parser.addSection("Expert options"); + + fnt = parser.getOption("--strict_highres_exp", "Resolution limit (in Angstrom) to restrict probability calculations in the expectation step", "OLD"); + if (fnt != "OLD") + strict_highres_exp = textToFloat(fnt); + + do_trust_ref_size = parser.checkOption("--trust_ref_size", "Trust the pixel and box size of the input reference; by default the program will die if these are different from the first optics group of the data"); + + // Debugging/analysis/hidden stuff + do_map = !checkParameter(argc, argv, "--no_map"); + minres_map = textToInteger(getParameter(argc, argv, "--minres_map", "5")); + abort_at_resolution = textToFloat(parser.getOption("--abort_at_resolution", "Abort when resolution reaches beyond this value", "-1", true)); + gridding_nr_iter = textToInteger(getParameter(argc, argv, "--gridding_iter", "10")); + debug1 = textToFloat(getParameter(argc, argv, "--debug1", "0.")); + debug2 = textToFloat(getParameter(argc, argv, "--debug2", "0.")); + debug3 = textToFloat(getParameter(argc, argv, "--debug3", "0.")); + do_bfactor = checkParameter(argc, argv, "--bfactor"); + // Read in initial sigmaNoise spectrum + fn_sigma = getParameter(argc, argv, "--sigma",""); + sigma2_fudge = textToFloat(getParameter(argc, argv, "--sigma2_fudge", "1.")); + do_acc_currentsize_despite_highres_exp = checkParameter(argc, argv, "--accuracy_current_size"); + do_sequential_halves_recons = checkParameter(argc, argv, "--sequential_halves_recons"); + do_always_join_random_halves = checkParameter(argc, argv, "--always_join_random_halves"); + do_use_all_data = checkParameter(argc, argv, "--use_all_data"); + do_always_cc = checkParameter(argc, argv, "--always_cc"); + do_only_sample_tilt = checkParameter(argc, argv, "--only_sample_tilt"); + minimum_angular_sampling = textToFloat(getParameter(argc, argv, "--minimum_angular_sampling", "0")); + maximum_angular_sampling = textToFloat(getParameter(argc, argv, "--maximum_angular_sampling", "0")); + asymmetric_padding = parser.checkOption("--asymmetric_padding", "", "false", true); + skip_gridding = !parser.checkOption("--dont_skip_gridding", "Perform gridding in the reconstruction step (obsolete?)"); + nr_iter_max = textToInteger(parser.getOption("--auto_iter_max", "In auto-refinement, stop at this iteration.", "999")); + debug_split_random_half = textToInteger(getParameter(argc, argv, "--debug_split_random_half", "0")); do_red = parser.checkOption("--do_red", "", "false", true); skip_realspace_helical_sym = parser.checkOption("--skip_realspace_helical_sym", "", "false", true); - // We read input optimiser set to create the output one - fn_OS = parser.getOption("--ios", "Input tomo optimiser set file. It is used to set --i, --ref or --solvent_mask if they are not provided. Updated output optimiser set is created.", ""); + // We read input optimiser set to create the output one + fn_OS = parser.getOption("--ios", "Input tomo optimiser set file. It is used to set --i, --ref or --solvent_mask if they are not provided. Updated output optimiser set is created.", ""); if (fn_OS != "") - { - optimisationSet.read(fn_OS); - } + { + optimisationSet.read(fn_OS); + } - do_print_metadata_labels = false; - do_print_symmetry_ops = false; + do_print_metadata_labels = false; + do_print_symmetry_ops = false; #ifdef DEBUG - std::cerr << "Leaving parseContinue" << std::endl; + std::cerr << "Leaving parseContinue" << std::endl; #endif } @@ -535,63 +535,63 @@ void MlOptimiser::parseInitial(int argc, char **argv) std::cerr<<"MlOptimiser::parseInitial Entering "< 0 ? tau2_fudge_arg : 1; - tau2_fudge_scheme = parser.getOption("--tau2_fudge_scheme", "Tau2 fudge factor updates scheme. Valid values are plain or -step. Where is the deflate factor during initial stage.",""); - mymodel.nr_classes = textToInteger(parser.getOption("--K", "Number of references to be refined", "1")); - particle_diameter = textToFloat(parser.getOption("--particle_diameter", "Diameter of the circular mask that will be applied to the experimental images (in Angstroms)", "-1")); - do_zero_mask = parser.checkOption("--zero_mask","Mask surrounding background in particles to zero (by default the solvent area is filled with random noise)"); - do_solvent = parser.checkOption("--flatten_solvent", "Perform masking on the references as well?"); - fn_mask = parser.getOption("--solvent_mask", "User-provided mask for the references (default is to use spherical mask with particle_diameter)", "None"); - fn_mask2 = parser.getOption("--solvent_mask2", "User-provided secondary mask (with its own average density)", "None"); - fn_lowpass_mask = parser.getOption("--lowpass_mask", "User-provided mask for low-pass filtering", "None"); - lowpass = textToFloat(parser.getOption("--lowpass", "User-provided cutoff for region specified above", "0")); - fn_tau = parser.getOption("--tau", "STAR file with input tau2-spectrum (to be kept constant)", "None"); - fn_local_symmetry = parser.getOption("--local_symmetry", "Local symmetry description file containing list of masks and their operators", "None"); - do_split_random_halves = parser.checkOption("--split_random_halves", "Refine two random halves of the data completely separately"); - low_resol_join_halves = textToFloat(parser.getOption("--low_resol_join_halves", "Resolution (in Angstrom) up to which the two random half-reconstructions will not be independent to prevent diverging orientations","-1")); - do_center_classes = parser.checkOption("--center_classes", "Re-center classes based on their center-of-mass?"); - - // Initialisation - int init_section = parser.addSection("Initialisation"); - fn_ref = parser.getOption("--ref", "Image, stack or star-file with the reference(s). (Compulsory for 3D refinement!)", "None"); - is_3d_model = parser.checkOption("--denovo_3dref", "Make an initial 3D model from randomly oriented 2D particles"); - mymodel.sigma2_offset = textToFloat(parser.getOption("--offset", "Initial estimated stddev for the origin offsets (in Angstroms)", "10")); - mymodel.sigma2_offset *= mymodel.sigma2_offset; - - // If tomo optimiser set is passed, we use it to initialise data, reference and mask - if (fn_OS != "") - { - optimisationSet.read(fn_OS); - - if (fn_data == "") - { - if (!optimisationSet.getValue(EMDL_TOMO_PARTICLES_FILE_NAME, fn_data)) - REPORT_ERROR("No particles filename was found in file " + fn_OS); - } + tau2_fudge_scheme = parser.getOption("--tau2_fudge_scheme", "Tau2 fudge factor updates scheme. Valid values are plain or -step. Where is the deflate factor during initial stage.",""); + mymodel.nr_classes = textToInteger(parser.getOption("--K", "Number of references to be refined", "1")); + particle_diameter = textToFloat(parser.getOption("--particle_diameter", "Diameter of the circular mask that will be applied to the experimental images (in Angstroms)", "-1")); + do_zero_mask = parser.checkOption("--zero_mask","Mask surrounding background in particles to zero (by default the solvent area is filled with random noise)"); + do_solvent = parser.checkOption("--flatten_solvent", "Perform masking on the references as well?"); + fn_mask = parser.getOption("--solvent_mask", "User-provided mask for the references (default is to use spherical mask with particle_diameter)", "None"); + fn_mask2 = parser.getOption("--solvent_mask2", "User-provided secondary mask (with its own average density)", "None"); + fn_lowpass_mask = parser.getOption("--lowpass_mask", "User-provided mask for low-pass filtering", "None"); + lowpass = textToFloat(parser.getOption("--lowpass", "User-provided cutoff for region specified above", "0")); + fn_tau = parser.getOption("--tau", "STAR file with input tau2-spectrum (to be kept constant)", "None"); + fn_local_symmetry = parser.getOption("--local_symmetry", "Local symmetry description file containing list of masks and their operators", "None"); + do_split_random_halves = parser.checkOption("--split_random_halves", "Refine two random halves of the data completely separately"); + low_resol_join_halves = textToFloat(parser.getOption("--low_resol_join_halves", "Resolution (in Angstrom) up to which the two random half-reconstructions will not be independent to prevent diverging orientations","-1")); + do_center_classes = parser.checkOption("--center_classes", "Re-center classes based on their center-of-mass?"); + + // Initialisation + int init_section = parser.addSection("Initialisation"); + fn_ref = parser.getOption("--ref", "Image, stack or star-file with the reference(s). (Compulsory for 3D refinement!)", "None"); + is_3d_model = parser.checkOption("--denovo_3dref", "Make an initial 3D model from randomly oriented 2D particles"); + mymodel.sigma2_offset = textToFloat(parser.getOption("--offset", "Initial estimated stddev for the origin offsets (in Angstroms)", "10")); + mymodel.sigma2_offset *= mymodel.sigma2_offset; + + // If tomo optimiser set is passed, we use it to initialise data, reference and mask + if (fn_OS != "") + { + optimisationSet.read(fn_OS); + + if (fn_data == "") + { + if (!optimisationSet.getValue(EMDL_TOMO_PARTICLES_FILE_NAME, fn_data)) + REPORT_ERROR("No particles filename was found in file " + fn_OS); + } if (fn_tomo == "") { if (!optimisationSet.getValue(EMDL_TOMO_TOMOGRAMS_FILE_NAME, fn_tomo)) @@ -602,353 +602,353 @@ void MlOptimiser::parseInitial(int argc, char **argv) if (!optimisationSet.getValue(EMDL_TOMO_TRAJECTORIES_FILE_NAME, fn_motion)) std::cout << " No motion trajectories were found in file " + fn_OS + ". Continuing without mask." << std::endl; } - if (fn_ref == "None") - { - if (!optimisationSet.getValue(EMDL_TOMO_REFERENCE_MAP_1_FILE_NAME, fn_ref)) - REPORT_ERROR("No reference half map filenames were found in file " + fn_OS); - } - if (fn_mask == "None") - { - if (!optimisationSet.getValue(EMDL_TOMO_REFERENCE_MASK_FILE_NAME, fn_mask)) - std::cout << " WARNING: No reference mask filename was found in file " + fn_OS + ". Continuing without mask." << std::endl; - } - } - - // Perform cross-product comparison at first iteration - do_firstiter_cc = parser.checkOption("--firstiter_cc", "Perform CC-calculation in the first iteration (use this if references are not on the absolute intensity scale)"); - ini_high = textToFloat(parser.getOption("--ini_high", "Resolution (in Angstroms) to which to limit refinement in the first iteration ", "-1")); - - // Set the orientations - int orientations_section = parser.addSection("Orientations"); - // Move these to sampling - adaptive_oversampling = textToInteger(parser.getOption("--oversampling", "Adaptive oversampling order to speed-up calculations (0=no oversampling, 1=2x, 2=4x, etc)", "1")); - sampling.healpix_order = textToInteger(parser.getOption("--healpix_order", "Healpix order for the angular sampling (before oversampling) on the (3D) sphere: hp2=15deg, hp3=7.5deg, etc", "2")); - sampling.psi_step = textToFloat(parser.getOption("--psi_step", "Sampling rate (before oversampling) for the in-plane angle (default=10deg for 2D, hp sampling for 3D)", "-1")); - sampling.limit_tilt = textToFloat(parser.getOption("--limit_tilt", "Limited tilt angle: positive for keeping side views, negative for keeping top views", "-91")); - - std::string sym_ = parser.getOption("--sym", "Symmetry group", "c1"); - - //Check if a comma separated list was provided - if (sym_.find(",") != std::string::npos) - { - std::stringstream ss(sym_); - std::string item; - while (std::getline(ss, item, ',')) - fn_multi_sym.push_back(item); - } - else - sampling.fn_sym = sym_; - - //Check for relax_symmetry option - std::string sym_relax_ = parser.getOption("--relax_sym", "Symmetry to be relaxed", ""); - sampling.fn_sym_relax = sym_relax_; - - sampling.offset_range = textToFloat(parser.getOption("--offset_range", "Search range for origin offsets (in pixels)", "6")); - sampling.offset_step = textToFloat(parser.getOption("--offset_step", "Sampling rate (before oversampling) for origin offsets (in pixels)", "2")); - // Jun19,2015 - Shaoda, Helical refinement - sampling.helical_offset_step = textToFloat(parser.getOption("--helical_offset_step", "Sampling rate (before oversampling) for offsets along helical axis (in Angstroms)", "-1")); - sampling.perturbation_factor = textToFloat(parser.getOption("--perturb", "Perturbation factor for the angular sampling (0=no perturb; 0.5=perturb)", "0.5")); - do_auto_refine = parser.checkOption("--auto_refine", "Perform 3D auto-refine procedure?"); - do_auto_sampling = parser.checkOption("--auto_sampling", "Perform auto-sampling (outside the 3D auto-refine procedure)?"); - autosampling_hporder_local_searches = textToInteger(parser.getOption("--auto_local_healpix_order", "Minimum healpix order (before oversampling) from which autosampling procedure will use local searches", "4")); - parser.setSection(orientations_section); - RFLOAT _sigma_ang = textToFloat(parser.getOption("--sigma_ang", "Stddev on all three Euler angles for local angular searches (of +/- 3 stddev)", "-1")); - RFLOAT _sigma_rot = textToFloat(parser.getOption("--sigma_rot", "Stddev on the first Euler angle for local angular searches (of +/- 3 stddev)", "-1")); - RFLOAT _sigma_tilt = textToFloat(parser.getOption("--sigma_tilt", "Stddev on the second Euler angle for local angular searches (of +/- 3 stddev)", "-1")); - RFLOAT _sigma_psi = textToFloat(parser.getOption("--sigma_psi", "Stddev on the in-plane angle for local angular searches (of +/- 3 stddev)", "-1")); - - if (_sigma_ang > 0.) - { - mymodel.orientational_prior_mode = PRIOR_ROTTILT_PSI; - // the sigma-values for the orientational prior are in model (and not in sampling) because one might like to estimate them - // from the data by calculating weighted sums of all angular differences: therefore it needs to be in wsum_model and thus in mymodel. - mymodel.sigma2_rot = mymodel.sigma2_tilt = mymodel.sigma2_psi = _sigma_ang * _sigma_ang; - } - else if (_sigma_rot > 0. || _sigma_tilt > 0. || _sigma_psi > 0.) - { - mymodel.orientational_prior_mode = PRIOR_ROTTILT_PSI; - mymodel.sigma2_rot = (_sigma_rot > 0. ) ? _sigma_rot * _sigma_rot : 0.; - mymodel.sigma2_tilt = (_sigma_tilt > 0.) ? _sigma_tilt * _sigma_tilt : 0.; - mymodel.sigma2_psi = (_sigma_psi > 0. ) ? _sigma_psi * _sigma_psi : 0.; - } - else - { - //default - // Very small to force the algorithm to take the current orientation - if (sym_relax_ != "") - { - mymodel.orientational_prior_mode = PRIOR_ROTTILT_PSI; - _sigma_ang = 0.0033; - mymodel.sigma2_rot = mymodel.sigma2_tilt = mymodel.sigma2_psi = _sigma_ang * _sigma_ang; - } - else - { - mymodel.orientational_prior_mode = NOPRIOR; - mymodel.sigma2_rot = mymodel.sigma2_tilt = mymodel.sigma2_psi = 0.; - } - } - do_skip_align = parser.checkOption("--skip_align", "Skip orientational assignment (only classify)?"); - do_skip_rotate = parser.checkOption("--skip_rotate", "Skip rotational assignment (only translate and classify)?"); - do_bimodal_psi = parser.checkOption("--bimodal_psi", "Do bimodal searches of psi angle?"); // Oct07,2015 - Shaoda, bimodal psi - do_skip_maximization = false; - - // Helical reconstruction - int helical_section = parser.addSection("Helical reconstruction (in development...)"); - do_helical_refine = parser.checkOption("--helix", "Perform 3D classification or refinement for helices?"); - ignore_helical_symmetry = parser.checkOption("--ignore_helical_symmetry", "Ignore helical symmetry?"); - mymodel.helical_nr_asu = textToInteger(parser.getOption("--helical_nr_asu", "Number of new helical asymmetric units (asu) per box (1 means no helical symmetry is present)", "1")); - helical_twist_initial = textToFloat(parser.getOption("--helical_twist_initial", "Helical twist (in degrees, positive values for right-handedness)", "0.")); - mymodel.helical_twist_min = textToFloat(parser.getOption("--helical_twist_min", "Minimum helical twist (in degrees, positive values for right-handedness)", "0.")); - mymodel.helical_twist_max = textToFloat(parser.getOption("--helical_twist_max", "Maximum helical twist (in degrees, positive values for right-handedness)", "0.")); - mymodel.helical_twist_inistep = textToFloat(parser.getOption("--helical_twist_inistep", "Initial step of helical twist search (in degrees)", "0.")); - helical_rise_initial = textToFloat(parser.getOption("--helical_rise_initial", "Helical rise (in Angstroms)", "0.")); - mymodel.helical_rise_min = textToFloat(parser.getOption("--helical_rise_min", "Minimum helical rise (in Angstroms)", "0.")); - mymodel.helical_rise_max = textToFloat(parser.getOption("--helical_rise_max", "Maximum helical rise (in Angstroms)", "0.")); - mymodel.helical_rise_inistep = textToFloat(parser.getOption("--helical_rise_inistep", "Initial step of helical rise search (in Angstroms)", "0.")); - helical_nstart = textToInteger(parser.getOption("--helical_nstart", "N-number for the N-start helix (only useful for rotational priors)", "1")); - helical_z_percentage = textToFloat(parser.getOption("--helical_z_percentage", "This box length along the center of Z axis contains good information of the helix. Important in imposing and refining symmetry", "0.3")); - helical_tube_inner_diameter = textToFloat(parser.getOption("--helical_inner_diameter", "Inner diameter of helical tubes in Angstroms (for masks of helical references and particles)", "-1.")); - helical_tube_outer_diameter = textToFloat(parser.getOption("--helical_outer_diameter", "Outer diameter of helical tubes in Angstroms (for masks of helical references and particles)", "-1.")); - do_helical_symmetry_local_refinement = parser.checkOption("--helical_symmetry_search", "Perform local refinement of helical symmetry?"); - helical_sigma_distance = textToFloat(parser.getOption("--helical_sigma_distance", "Sigma of distance along the helical tracks", "-1.")); - helical_keep_tilt_prior_fixed = parser.checkOption("--helical_keep_tilt_prior_fixed", "Keep helical tilt priors fixed (at 90 degrees) in global angular searches?"); - if (ignore_helical_symmetry) - { - mymodel.helical_nr_asu = 1; // IMPORTANT ! - do_helical_symmetry_local_refinement = false; - helical_twist_initial = mymodel.helical_twist_min = mymodel.helical_twist_max = mymodel.helical_twist_inistep = 0.; - helical_rise_initial = mymodel.helical_rise_min = mymodel.helical_rise_max = mymodel.helical_rise_inistep = 0.; - helical_z_percentage = 0.; - } - mymodel.initialiseHelicalParametersLists(helical_twist_initial, helical_rise_initial); - mymodel.is_helix = do_helical_refine; - RFLOAT tmp_RFLOAT = 0.; - if (mymodel.helical_rise_min > mymodel.helical_rise_max) - SWAP(mymodel.helical_rise_min, mymodel.helical_rise_max, tmp_RFLOAT); - if (mymodel.helical_twist_min > mymodel.helical_twist_max) - SWAP(mymodel.helical_twist_min, mymodel.helical_twist_max, tmp_RFLOAT); - helical_fourier_mask_resols = parser.getOption("--helical_exclude_resols", "Resolutions (in A) along helical axis to exclude from refinement (comma-separated pairs, e.g. 50,5)", ""); - fn_fourier_mask = parser.getOption("--fourier_mask", "Originally-sized, FFTW-centred image with Fourier mask for Projector", "None"); - - // CTF, norm, scale, bfactor correction etc. - int corrections_section = parser.addSection("Corrections"); - do_ctf_correction = parser.checkOption("--ctf", "Perform CTF correction?"); - do_ctf_padding = parser.checkOption("--pad_ctf", "Perform CTF padding to treat CTF aliaising better?"); - if (do_ctf_padding) - REPORT_ERROR("--pad_ctf currently disabled."); - intact_ctf_first_peak = parser.checkOption("--ctf_intact_first_peak", "Ignore CTFs until their first peak?"); - refs_are_ctf_corrected = !parser.checkOption("--ctf_uncorrected_ref", "Have the input references not been CTF-amplitude corrected?"); - if (checkParameter(argc, argv, "--ctf_corrected_ref")) - std::cerr << "Warning: the option --ctf_corrected_ref has been removed. By default, this is assumed to be true. If the reference is not CTF corrected, use --ctf_uncorrected_ref.\n" << std::endl; - - ctf_phase_flipped = parser.checkOption("--ctf_phase_flipped", "Have the data been CTF phase-flipped?"); - only_flip_phases = parser.checkOption("--only_flip_phases", "Only perform CTF phase-flipping? (default is full amplitude-correction)"); - do_norm_correction = parser.checkOption("--norm", "Perform normalisation-error correction?"); - do_scale_correction = parser.checkOption("--scale", "Perform intensity-scale corrections on image groups?"); - // Allow switching off norm and scale (which is on by default in the GUI) - if (parser.checkOption("--no_norm", "Switch off normalisation-error correction?")) - do_norm_correction = false; - if (parser.checkOption("--no_scale", "Switch off intensity-scale corrections on image groups?")) - do_scale_correction = false; - - // SGD stuff - int grad_section = parser.addSection("Stochastic Gradient Descent"); - gradient_refine = parser.checkOption("--grad", "Perform gradient based optimisation (instead of default expectation-maximization)"); - grad_em_iters = textToInteger(parser.getOption("--grad_em_iters", "Number of iterations at the end of a gradient refinement using Expectation-Maximization", "0")); - // Stochastic EM is implemented as a variant of SGD, though it is really a different algorithm! - - grad_ini_frac = textToFloat(parser.getOption("--grad_ini_frac", "Fraction of iterations in the initial phase of refinement", "0.3")); - grad_fin_frac = textToFloat(parser.getOption("--grad_fin_frac", "Fraction of iterations in the final phase of refinement", "0.2")); - - if (grad_ini_frac <= 0 || 1 <= grad_ini_frac) - REPORT_ERROR("Invalid value for --grad_ini_frac."); - if (grad_fin_frac <= 0 || 1 <= grad_fin_frac) - REPORT_ERROR("Invalid value for --grad_fin_frac."); - - if (grad_ini_frac + grad_fin_frac > 0.9) { - RFLOAT sum = grad_ini_frac + grad_fin_frac + 0.1; - grad_ini_frac /= sum; - grad_fin_frac /= sum; - } - grad_ini_iter = nr_iter * grad_ini_frac; - grad_fin_iter = nr_iter * grad_fin_frac; - grad_inbetween_iter = nr_iter - grad_ini_iter - grad_fin_iter; - if (grad_inbetween_iter < 0) - grad_inbetween_iter = 0; - - grad_min_resol = textToInteger(parser.getOption("--grad_min_resol", "Adjusting the signal under-estimation during gradient optimization to this resolution.", "20")); - grad_ini_resol = textToInteger(parser.getOption("--grad_ini_resol", "Resolution cutoff during the initial gradient refinement iterations (A)", "-1")); - grad_fin_resol = textToInteger(parser.getOption("--grad_fin_resol", "Resolution cutoff during the final gradient refinement iterations (A)", "-1")); - grad_ini_subset_size = textToInteger(parser.getOption("--grad_ini_subset", "Mini-batch size during the initial gradient refinement iterations", "-1")); - grad_fin_subset_size = textToInteger(parser.getOption("--grad_fin_subset", "Mini-batch size during the final gradient refinement iterations", "-1")); - mu = textToFloat(parser.getOption("--mu", "Momentum parameter for gradient refinement updates", "0.9")); - - grad_stepsize = textToFloat(parser.getOption("--grad_stepsize", "Step size parameter for gradient optimisation.", "-1")); - grad_stepsize_scheme = parser.getOption("--grad_stepsize_scheme", - "Gradient step size updates scheme. Valid values are plain or -step . Where is the initial inflate.",""); - - write_every_grad_iter = textToInteger(parser.getOption("--grad_write_iter", "Write out model every so many iterations in SGD (default is writing out all iters)", "10")); - maximum_significants_arg = textToInteger(parser.getOption("--maxsig", "Maximum number of most significant poses & translations to consider", "-1")); - do_init_blobs = !parser.checkOption("--no_init_blobs", "Use this to switch off initializing models with random Gaussians (which is new in relion-4.0)."); - do_som = parser.checkOption("--som", "Calculate self-organizing map instead of classification."); - som_starting_nodes = textToInteger(parser.getOption("--som_ini_nodes", "Number of initial SOM nodes.", "2")); - som_connectivity = textToFloat(parser.getOption("--som_connectivity", "Number of average active neighbour connections.", "5.0")); - som_inactivity_threshold = textToFloat(parser.getOption("--som_inactivity_threshold", "Threshold for inactivity before node is dropped.", "0.01")); - som_neighbour_pull = textToFloat(parser.getOption("--som_neighbour_pull", "Portion of gradient applied to connected nodes.", "0.2")); - class_inactivity_threshold = textToFloat(parser.getOption("--class_inactivity_threshold", "Replace classes with little activity during gradient based classification.", "0")); - - if (do_som && !gradient_refine) - REPORT_ERROR("SOM can only be calculated with a gradient optimization."); - - if (do_som && mymodel.nr_classes < 3) - REPORT_ERROR("Too low maximum class limit for SOM calculations."); - - if (do_som && som_starting_nodes > mymodel.nr_classes) - REPORT_ERROR("Cannot initiate more nodes than maximum number of nodes."); - - if (som_neighbour_pull < 0 || 1 <= som_neighbour_pull) - REPORT_ERROR("--som_neighbour_pull should be more then or equal to zero and less than one."); - - // Subtomo Avg stuff - int subtomogram_section = parser.addSection("Subtomogram averaging"); - normalised_subtomos = parser.checkOption("--normalised_subtomo", "Have subtomograms been multiplicity normalised? (Default=False)"); - do_skip_subtomo_correction = parser.checkOption("--skip_subtomo_multi", "Skip subtomo multiplicity correction"); - ctf3d_squared = !parser.checkOption("--ctf3d_not_squared", "CTF3D files contain sqrt(CTF^2) patterns"); - subtomo_multi_thr = textToFloat(parser.getOption("--subtomo_multi_thr", "Threshold to remove marginal voxels during expectation", "0.01")); - - // Computation stuff - // The number of threads is always read from the command line - int computation_section = parser.addSection("Computation"); - x_pool = textToInteger(parser.getOption("--pool", "Number of images to pool for each thread task", "1")); - nr_threads = textToInteger(parser.getOption("--j", "Number of threads to run in parallel (only useful on multi-core machines)", "1")); - combine_weights_thru_disc = !parser.checkOption("--dont_combine_weights_via_disc", "Send the large arrays of summed weights through the MPI network, instead of writing large files to disc"); - do_shifts_onthefly = parser.checkOption("--onthefly_shifts", "Calculate shifted images on-the-fly, do not store precalculated ones in memory"); - do_parallel_disc_io = !parser.checkOption("--no_parallel_disc_io", "Do NOT let parallel (MPI) processes access the disc simultaneously (use this option with NFS)"); - do_preread_images = parser.checkOption("--preread_images", "Use this to let the leader process read all particles into memory. Be careful you have enough RAM for large data sets!"); - fn_scratch = parser.getOption("--scratch_dir", "If provided, particle stacks will be copied to this local scratch disk prior to refinement.", ""); - keep_free_scratch_Gb = textToFloat(parser.getOption("--keep_free_scratch", "Space available for copying particle stacks (in Gb)", "10")); - do_reuse_scratch = parser.checkOption("--reuse_scratch", "Re-use data on scratchdir, instead of wiping it and re-copying all data."); - keep_scratch = parser.checkOption("--keep_scratch", "Don't remove scratch after convergence. Following jobs that use EXACTLY the same particles should use --reuse_scratch."); - do_fast_subsets = parser.checkOption("--fast_subsets", "Use faster optimisation by using subsets of the data in the first 15 iterations"); + if (fn_ref == "None") + { + if (!optimisationSet.getValue(EMDL_TOMO_REFERENCE_MAP_1_FILE_NAME, fn_ref)) + REPORT_ERROR("No reference half map filenames were found in file " + fn_OS); + } + if (fn_mask == "None") + { + if (!optimisationSet.getValue(EMDL_TOMO_REFERENCE_MASK_FILE_NAME, fn_mask)) + std::cout << " WARNING: No reference mask filename was found in file " + fn_OS + ". Continuing without mask." << std::endl; + } + } + + // Perform cross-product comparison at first iteration + do_firstiter_cc = parser.checkOption("--firstiter_cc", "Perform CC-calculation in the first iteration (use this if references are not on the absolute intensity scale)"); + ini_high = textToFloat(parser.getOption("--ini_high", "Resolution (in Angstroms) to which to limit refinement in the first iteration ", "-1")); + + // Set the orientations + int orientations_section = parser.addSection("Orientations"); + // Move these to sampling + adaptive_oversampling = textToInteger(parser.getOption("--oversampling", "Adaptive oversampling order to speed-up calculations (0=no oversampling, 1=2x, 2=4x, etc)", "1")); + sampling.healpix_order = textToInteger(parser.getOption("--healpix_order", "Healpix order for the angular sampling (before oversampling) on the (3D) sphere: hp2=15deg, hp3=7.5deg, etc", "2")); + sampling.psi_step = textToFloat(parser.getOption("--psi_step", "Sampling rate (before oversampling) for the in-plane angle (default=10deg for 2D, hp sampling for 3D)", "-1")); + sampling.limit_tilt = textToFloat(parser.getOption("--limit_tilt", "Limited tilt angle: positive for keeping side views, negative for keeping top views", "-91")); + + std::string sym_ = parser.getOption("--sym", "Symmetry group", "c1"); + + //Check if a comma separated list was provided + if (sym_.find(",") != std::string::npos) + { + std::stringstream ss(sym_); + std::string item; + while (std::getline(ss, item, ',')) + fn_multi_sym.push_back(item); + } + else + sampling.fn_sym = sym_; + + //Check for relax_symmetry option + std::string sym_relax_ = parser.getOption("--relax_sym", "Symmetry to be relaxed", ""); + sampling.fn_sym_relax = sym_relax_; + + sampling.offset_range = textToFloat(parser.getOption("--offset_range", "Search range for origin offsets (in pixels)", "6")); + sampling.offset_step = textToFloat(parser.getOption("--offset_step", "Sampling rate (before oversampling) for origin offsets (in pixels)", "2")); + // Jun19,2015 - Shaoda, Helical refinement + sampling.helical_offset_step = textToFloat(parser.getOption("--helical_offset_step", "Sampling rate (before oversampling) for offsets along helical axis (in Angstroms)", "-1")); + sampling.perturbation_factor = textToFloat(parser.getOption("--perturb", "Perturbation factor for the angular sampling (0=no perturb; 0.5=perturb)", "0.5")); + do_auto_refine = parser.checkOption("--auto_refine", "Perform 3D auto-refine procedure?"); + do_auto_sampling = parser.checkOption("--auto_sampling", "Perform auto-sampling (outside the 3D auto-refine procedure)?"); + autosampling_hporder_local_searches = textToInteger(parser.getOption("--auto_local_healpix_order", "Minimum healpix order (before oversampling) from which autosampling procedure will use local searches", "4")); + parser.setSection(orientations_section); + RFLOAT _sigma_ang = textToFloat(parser.getOption("--sigma_ang", "Stddev on all three Euler angles for local angular searches (of +/- 3 stddev)", "-1")); + RFLOAT _sigma_rot = textToFloat(parser.getOption("--sigma_rot", "Stddev on the first Euler angle for local angular searches (of +/- 3 stddev)", "-1")); + RFLOAT _sigma_tilt = textToFloat(parser.getOption("--sigma_tilt", "Stddev on the second Euler angle for local angular searches (of +/- 3 stddev)", "-1")); + RFLOAT _sigma_psi = textToFloat(parser.getOption("--sigma_psi", "Stddev on the in-plane angle for local angular searches (of +/- 3 stddev)", "-1")); + + if (_sigma_ang > 0.) + { + mymodel.orientational_prior_mode = PRIOR_ROTTILT_PSI; + // the sigma-values for the orientational prior are in model (and not in sampling) because one might like to estimate them + // from the data by calculating weighted sums of all angular differences: therefore it needs to be in wsum_model and thus in mymodel. + mymodel.sigma2_rot = mymodel.sigma2_tilt = mymodel.sigma2_psi = _sigma_ang * _sigma_ang; + } + else if (_sigma_rot > 0. || _sigma_tilt > 0. || _sigma_psi > 0.) + { + mymodel.orientational_prior_mode = PRIOR_ROTTILT_PSI; + mymodel.sigma2_rot = (_sigma_rot > 0. ) ? _sigma_rot * _sigma_rot : 0.; + mymodel.sigma2_tilt = (_sigma_tilt > 0.) ? _sigma_tilt * _sigma_tilt : 0.; + mymodel.sigma2_psi = (_sigma_psi > 0. ) ? _sigma_psi * _sigma_psi : 0.; + } + else + { + //default + // Very small to force the algorithm to take the current orientation + if (sym_relax_ != "") + { + mymodel.orientational_prior_mode = PRIOR_ROTTILT_PSI; + _sigma_ang = 0.0033; + mymodel.sigma2_rot = mymodel.sigma2_tilt = mymodel.sigma2_psi = _sigma_ang * _sigma_ang; + } + else + { + mymodel.orientational_prior_mode = NOPRIOR; + mymodel.sigma2_rot = mymodel.sigma2_tilt = mymodel.sigma2_psi = 0.; + } + } + do_skip_align = parser.checkOption("--skip_align", "Skip orientational assignment (only classify)?"); + do_skip_rotate = parser.checkOption("--skip_rotate", "Skip rotational assignment (only translate and classify)?"); + do_bimodal_psi = parser.checkOption("--bimodal_psi", "Do bimodal searches of psi angle?"); // Oct07,2015 - Shaoda, bimodal psi + do_skip_maximization = false; + + // Helical reconstruction + int helical_section = parser.addSection("Helical reconstruction (in development...)"); + do_helical_refine = parser.checkOption("--helix", "Perform 3D classification or refinement for helices?"); + ignore_helical_symmetry = parser.checkOption("--ignore_helical_symmetry", "Ignore helical symmetry?"); + mymodel.helical_nr_asu = textToInteger(parser.getOption("--helical_nr_asu", "Number of new helical asymmetric units (asu) per box (1 means no helical symmetry is present)", "1")); + helical_twist_initial = textToFloat(parser.getOption("--helical_twist_initial", "Helical twist (in degrees, positive values for right-handedness)", "0.")); + mymodel.helical_twist_min = textToFloat(parser.getOption("--helical_twist_min", "Minimum helical twist (in degrees, positive values for right-handedness)", "0.")); + mymodel.helical_twist_max = textToFloat(parser.getOption("--helical_twist_max", "Maximum helical twist (in degrees, positive values for right-handedness)", "0.")); + mymodel.helical_twist_inistep = textToFloat(parser.getOption("--helical_twist_inistep", "Initial step of helical twist search (in degrees)", "0.")); + helical_rise_initial = textToFloat(parser.getOption("--helical_rise_initial", "Helical rise (in Angstroms)", "0.")); + mymodel.helical_rise_min = textToFloat(parser.getOption("--helical_rise_min", "Minimum helical rise (in Angstroms)", "0.")); + mymodel.helical_rise_max = textToFloat(parser.getOption("--helical_rise_max", "Maximum helical rise (in Angstroms)", "0.")); + mymodel.helical_rise_inistep = textToFloat(parser.getOption("--helical_rise_inistep", "Initial step of helical rise search (in Angstroms)", "0.")); + helical_nstart = textToInteger(parser.getOption("--helical_nstart", "N-number for the N-start helix (only useful for rotational priors)", "1")); + helical_z_percentage = textToFloat(parser.getOption("--helical_z_percentage", "This box length along the center of Z axis contains good information of the helix. Important in imposing and refining symmetry", "0.3")); + helical_tube_inner_diameter = textToFloat(parser.getOption("--helical_inner_diameter", "Inner diameter of helical tubes in Angstroms (for masks of helical references and particles)", "-1.")); + helical_tube_outer_diameter = textToFloat(parser.getOption("--helical_outer_diameter", "Outer diameter of helical tubes in Angstroms (for masks of helical references and particles)", "-1.")); + do_helical_symmetry_local_refinement = parser.checkOption("--helical_symmetry_search", "Perform local refinement of helical symmetry?"); + helical_sigma_distance = textToFloat(parser.getOption("--helical_sigma_distance", "Sigma of distance along the helical tracks", "-1.")); + helical_keep_tilt_prior_fixed = parser.checkOption("--helical_keep_tilt_prior_fixed", "Keep helical tilt priors fixed (at 90 degrees) in global angular searches?"); + if (ignore_helical_symmetry) + { + mymodel.helical_nr_asu = 1; // IMPORTANT ! + do_helical_symmetry_local_refinement = false; + helical_twist_initial = mymodel.helical_twist_min = mymodel.helical_twist_max = mymodel.helical_twist_inistep = 0.; + helical_rise_initial = mymodel.helical_rise_min = mymodel.helical_rise_max = mymodel.helical_rise_inistep = 0.; + helical_z_percentage = 0.; + } + mymodel.initialiseHelicalParametersLists(helical_twist_initial, helical_rise_initial); + mymodel.is_helix = do_helical_refine; + RFLOAT tmp_RFLOAT = 0.; + if (mymodel.helical_rise_min > mymodel.helical_rise_max) + SWAP(mymodel.helical_rise_min, mymodel.helical_rise_max, tmp_RFLOAT); + if (mymodel.helical_twist_min > mymodel.helical_twist_max) + SWAP(mymodel.helical_twist_min, mymodel.helical_twist_max, tmp_RFLOAT); + helical_fourier_mask_resols = parser.getOption("--helical_exclude_resols", "Resolutions (in A) along helical axis to exclude from refinement (comma-separated pairs, e.g. 50,5)", ""); + fn_fourier_mask = parser.getOption("--fourier_mask", "Originally-sized, FFTW-centred image with Fourier mask for Projector", "None"); + + // CTF, norm, scale, bfactor correction etc. + int corrections_section = parser.addSection("Corrections"); + do_ctf_correction = parser.checkOption("--ctf", "Perform CTF correction?"); + do_ctf_padding = parser.checkOption("--pad_ctf", "Perform CTF padding to treat CTF aliaising better?"); + if (do_ctf_padding) + REPORT_ERROR("--pad_ctf currently disabled."); + intact_ctf_first_peak = parser.checkOption("--ctf_intact_first_peak", "Ignore CTFs until their first peak?"); + refs_are_ctf_corrected = !parser.checkOption("--ctf_uncorrected_ref", "Have the input references not been CTF-amplitude corrected?"); + if (checkParameter(argc, argv, "--ctf_corrected_ref")) + std::cerr << "Warning: the option --ctf_corrected_ref has been removed. By default, this is assumed to be true. If the reference is not CTF corrected, use --ctf_uncorrected_ref.\n" << std::endl; + + ctf_phase_flipped = parser.checkOption("--ctf_phase_flipped", "Have the data been CTF phase-flipped?"); + only_flip_phases = parser.checkOption("--only_flip_phases", "Only perform CTF phase-flipping? (default is full amplitude-correction)"); + do_norm_correction = parser.checkOption("--norm", "Perform normalisation-error correction?"); + do_scale_correction = parser.checkOption("--scale", "Perform intensity-scale corrections on image groups?"); + // Allow switching off norm and scale (which is on by default in the GUI) + if (parser.checkOption("--no_norm", "Switch off normalisation-error correction?")) + do_norm_correction = false; + if (parser.checkOption("--no_scale", "Switch off intensity-scale corrections on image groups?")) + do_scale_correction = false; + + // SGD stuff + int grad_section = parser.addSection("Stochastic Gradient Descent"); + gradient_refine = parser.checkOption("--grad", "Perform gradient based optimisation (instead of default expectation-maximization)"); + grad_em_iters = textToInteger(parser.getOption("--grad_em_iters", "Number of iterations at the end of a gradient refinement using Expectation-Maximization", "0")); + // Stochastic EM is implemented as a variant of SGD, though it is really a different algorithm! + + grad_ini_frac = textToFloat(parser.getOption("--grad_ini_frac", "Fraction of iterations in the initial phase of refinement", "0.3")); + grad_fin_frac = textToFloat(parser.getOption("--grad_fin_frac", "Fraction of iterations in the final phase of refinement", "0.2")); + + if (grad_ini_frac <= 0 || 1 <= grad_ini_frac) + REPORT_ERROR("Invalid value for --grad_ini_frac."); + if (grad_fin_frac <= 0 || 1 <= grad_fin_frac) + REPORT_ERROR("Invalid value for --grad_fin_frac."); + + if (grad_ini_frac + grad_fin_frac > 0.9) { + RFLOAT sum = grad_ini_frac + grad_fin_frac + 0.1; + grad_ini_frac /= sum; + grad_fin_frac /= sum; + } + grad_ini_iter = nr_iter * grad_ini_frac; + grad_fin_iter = nr_iter * grad_fin_frac; + grad_inbetween_iter = nr_iter - grad_ini_iter - grad_fin_iter; + if (grad_inbetween_iter < 0) + grad_inbetween_iter = 0; + + grad_min_resol = textToInteger(parser.getOption("--grad_min_resol", "Adjusting the signal under-estimation during gradient optimization to this resolution.", "20")); + grad_ini_resol = textToInteger(parser.getOption("--grad_ini_resol", "Resolution cutoff during the initial gradient refinement iterations (A)", "-1")); + grad_fin_resol = textToInteger(parser.getOption("--grad_fin_resol", "Resolution cutoff during the final gradient refinement iterations (A)", "-1")); + grad_ini_subset_size = textToInteger(parser.getOption("--grad_ini_subset", "Mini-batch size during the initial gradient refinement iterations", "-1")); + grad_fin_subset_size = textToInteger(parser.getOption("--grad_fin_subset", "Mini-batch size during the final gradient refinement iterations", "-1")); + mu = textToFloat(parser.getOption("--mu", "Momentum parameter for gradient refinement updates", "0.9")); + + grad_stepsize = textToFloat(parser.getOption("--grad_stepsize", "Step size parameter for gradient optimisation.", "-1")); + grad_stepsize_scheme = parser.getOption("--grad_stepsize_scheme", + "Gradient step size updates scheme. Valid values are plain or -step . Where is the initial inflate.",""); + + write_every_grad_iter = textToInteger(parser.getOption("--grad_write_iter", "Write out model every so many iterations in SGD (default is writing out all iters)", "10")); + maximum_significants_arg = textToInteger(parser.getOption("--maxsig", "Maximum number of most significant poses & translations to consider", "-1")); + do_init_blobs = !parser.checkOption("--no_init_blobs", "Use this to switch off initializing models with random Gaussians (which is new in relion-4.0)."); + do_som = parser.checkOption("--som", "Calculate self-organizing map instead of classification."); + som_starting_nodes = textToInteger(parser.getOption("--som_ini_nodes", "Number of initial SOM nodes.", "2")); + som_connectivity = textToFloat(parser.getOption("--som_connectivity", "Number of average active neighbour connections.", "5.0")); + som_inactivity_threshold = textToFloat(parser.getOption("--som_inactivity_threshold", "Threshold for inactivity before node is dropped.", "0.01")); + som_neighbour_pull = textToFloat(parser.getOption("--som_neighbour_pull", "Portion of gradient applied to connected nodes.", "0.2")); + class_inactivity_threshold = textToFloat(parser.getOption("--class_inactivity_threshold", "Replace classes with little activity during gradient based classification.", "0")); + + if (do_som && !gradient_refine) + REPORT_ERROR("SOM can only be calculated with a gradient optimization."); + + if (do_som && mymodel.nr_classes < 3) + REPORT_ERROR("Too low maximum class limit for SOM calculations."); + + if (do_som && som_starting_nodes > mymodel.nr_classes) + REPORT_ERROR("Cannot initiate more nodes than maximum number of nodes."); + + if (som_neighbour_pull < 0 || 1 <= som_neighbour_pull) + REPORT_ERROR("--som_neighbour_pull should be more then or equal to zero and less than one."); + + // Subtomo Avg stuff + int subtomogram_section = parser.addSection("Subtomogram averaging"); + normalised_subtomos = parser.checkOption("--normalised_subtomo", "Have subtomograms been multiplicity normalised? (Default=False)"); + do_skip_subtomo_correction = parser.checkOption("--skip_subtomo_multi", "Skip subtomo multiplicity correction"); + ctf3d_squared = !parser.checkOption("--ctf3d_not_squared", "CTF3D files contain sqrt(CTF^2) patterns"); + subtomo_multi_thr = textToFloat(parser.getOption("--subtomo_multi_thr", "Threshold to remove marginal voxels during expectation", "0.01")); + + // Computation stuff + // The number of threads is always read from the command line + int computation_section = parser.addSection("Computation"); + x_pool = textToInteger(parser.getOption("--pool", "Number of images to pool for each thread task", "1")); + nr_threads = textToInteger(parser.getOption("--j", "Number of threads to run in parallel (only useful on multi-core machines)", "1")); + combine_weights_thru_disc = !parser.checkOption("--dont_combine_weights_via_disc", "Send the large arrays of summed weights through the MPI network, instead of writing large files to disc"); + do_shifts_onthefly = parser.checkOption("--onthefly_shifts", "Calculate shifted images on-the-fly, do not store precalculated ones in memory"); + do_parallel_disc_io = !parser.checkOption("--no_parallel_disc_io", "Do NOT let parallel (MPI) processes access the disc simultaneously (use this option with NFS)"); + do_preread_images = parser.checkOption("--preread_images", "Use this to let the leader process read all particles into memory. Be careful you have enough RAM for large data sets!"); + fn_scratch = parser.getOption("--scratch_dir", "If provided, particle stacks will be copied to this local scratch disk prior to refinement.", ""); + keep_free_scratch_Gb = textToFloat(parser.getOption("--keep_free_scratch", "Space available for copying particle stacks (in Gb)", "10")); + do_reuse_scratch = parser.checkOption("--reuse_scratch", "Re-use data on scratchdir, instead of wiping it and re-copying all data."); + keep_scratch = parser.checkOption("--keep_scratch", "Don't remove scratch after convergence. Following jobs that use EXACTLY the same particles should use --reuse_scratch."); + do_fast_subsets = parser.checkOption("--fast_subsets", "Use faster optimisation by using subsets of the data in the first 15 iterations"); #ifdef ALTCPU - do_cpu = parser.checkOption("--cpu", "Use intel vectorisation implementation for CPU"); + do_cpu = parser.checkOption("--cpu", "Use intel vectorisation implementation for CPU"); #else do_cpu = false; #endif - do_gpu = parser.checkOption("--gpu", "Use available gpu resources for some calculations"); - gpu_ids = parser.getOption("--gpu", "Device ids for each MPI-thread","default"); + do_gpu = parser.checkOption("--gpu", "Use available gpu resources for some calculations"); + gpu_ids = parser.getOption("--gpu", "Device ids for each MPI-thread","default"); #ifndef _CUDA_ENABLED if(do_gpu) - { - std::cerr << "+ WARNING : Relion was compiled without CUDA of at least version 7.0 - you do NOT have support for GPUs" << std::endl; - do_gpu = false; - } -#endif - double temp_reqSize = textToDouble(parser.getOption("--free_gpu_memory", "GPU device memory (in Mb) to leave free after allocation.", "0")); - if(!do_zero_mask) - temp_reqSize += 100; - temp_reqSize *= 1000*1000; - if(temp_reqSize<0) - REPORT_ERROR("Invalid free_gpu_memory value."); - else - requested_free_gpu_memory = temp_reqSize; - - // Expert options - int expert_section = parser.addSection("Expert options"); - mymodel.padding_factor = textToFloat(parser.getOption("--pad", "Oversampling factor for the Fourier transforms of the references", "2")); - - ref_angpix = textToFloat(parser.getOption("--ref_angpix", "Pixel size (in A) for the input reference (default is to read from header)", "-1.")); - mymodel.interpolator = (parser.checkOption("--NN", "Perform nearest-neighbour instead of linear Fourier-space interpolation?")) ? NEAREST_NEIGHBOUR : TRILINEAR; - mymodel.r_min_nn = textToInteger(parser.getOption("--r_min_nn", "Minimum number of Fourier shells to perform linear Fourier-space interpolation", "10")); - verb = textToInteger(parser.getOption("--verb", "Verbosity (1=normal, 0=silent)", "1")); - random_seed = textToInteger(parser.getOption("--random_seed", "Number for the random seed generator", "-1")); - max_coarse_size = textToInteger(parser.getOption("--coarse_size", "Maximum image size for the first pass of the adaptive sampling approach", "-1")); - adaptive_fraction = textToFloat(parser.getOption("--adaptive_fraction", "Fraction of the weights to be considered in the first pass of adaptive oversampling ", "0.999")); - width_mask_edge = textToInteger(parser.getOption("--maskedge", "Width of the soft edge of the spherical mask (in pixels)", "5")); - // If we're doing helical, and maskedge is not given, use a default maskedge of 10 - if (helical_tube_outer_diameter > 0. && !checkParameter(argc, argv, "--maskedge")) width_mask_edge = 10.; - fix_sigma_noise = parser.checkOption("--fix_sigma_noise", "Fix the experimental noise spectra?"); - fix_sigma_offset = parser.checkOption("--fix_sigma_offset", "Fix the stddev in the origin offsets?"); - incr_size = textToInteger(parser.getOption("--incr_size", "Number of Fourier shells beyond the current resolution to be included in refinement", "10")); - do_print_metadata_labels = parser.checkOption("--print_metadata_labels", "Print a table with definitions of all metadata labels, and exit"); - do_print_symmetry_ops = parser.checkOption("--print_symmetry_ops", "Print all symmetry transformation matrices, and exit"); - strict_highres_exp = textToFloat(parser.getOption("--strict_highres_exp", "High resolution limit (in Angstrom) to restrict probability calculations in the expectation step", "-1")); - strict_lowres_exp = textToFloat(parser.getOption("--strict_lowres_exp", "Low resolution limit (in Angstrom) to restrict probability calculations in the expectation step", "-1")); - dont_raise_norm_error = parser.checkOption("--dont_check_norm", "Skip the check whether the images are normalised correctly"); - do_always_cc = parser.checkOption("--always_cc", "Perform CC-calculation in all iterations (useful for faster denovo model generation?)"); - do_phase_random_fsc = parser.checkOption("--solvent_correct_fsc", "Correct FSC curve for the effects of the solvent mask?"); - do_skip_maximization = parser.checkOption("--skip_maximize", "Skip maximization step (only write out data.star file)?"); - failsafe_threshold = textToInteger(parser.getOption("--failsafe_threshold", "Maximum number of particles permitted to be handled by fail-safe mode, due to zero sum of weights, before exiting with an error (GPU only).", "40")); - do_external_reconstruct = parser.checkOption("--external_reconstruct", "Perform the reconstruction step outside relion_refine, e.g. for learned priors?)"); - nr_iter_max = textToInteger(parser.getOption("--auto_iter_max", "In auto-refinement, stop at this iteration.", "999")); - auto_ignore_angle_changes = parser.checkOption("--auto_ignore_angles", "In auto-refinement, update angular sampling regardless of changes in orientations for convergence. This makes convergence faster."); - auto_resolution_based_angles= parser.checkOption("--auto_resol_angles", "In auto-refinement, update angular sampling based on resolution-based required sampling. This makes convergence faster."); - allow_coarser_samplings = parser.checkOption("--allow_coarser_sampling", "In 2D/3D classification, allow coarser angular and translational samplings if accuracies are bad (typically in earlier iterations."); - do_trust_ref_size = parser.checkOption("--trust_ref_size", "Trust the pixel and box size of the input reference; by default the program will die if these are different from the first optics group of the data"); - minimum_nr_particles_sigma2_noise = textToInteger(parser.getOption("--nr_parts_sigma2noise", "Number of particles (per optics group) for initial noise spectra estimation.", "1000")); - ///////////////// Special stuff for first iteration (only accessible via CL, not through readSTAR //////////////////// - - // When reading from the CL: always start at iteration 1 and subset 1 - iter = 0; - // When starting from CL: always calculate initial sigma_noise - do_calculate_initial_sigma_noise = true; - // Start average norm correction at 1! - mymodel.avg_norm_correction = 1.; - // Always initialise the PDF of the directions - directions_have_changed = true; - - // Only reconstruct and join random halves are only available when continuing an old run - do_join_random_halves = false; - - // For auto-sampling and convergence check - nr_iter_wo_resol_gain = 0; - nr_iter_wo_large_hidden_variable_changes = 0; - current_changes_optimal_classes = 9999999; - current_changes_optimal_offsets = 999.; - current_changes_optimal_orientations = 999.; - smallest_changes_optimal_classes = 9999999; - smallest_changes_optimal_offsets = 999.; - smallest_changes_optimal_orientations = 999.; - acc_rot = acc_trans = 999.; - - best_resol_thus_far = 1./999.; - has_converged = false; - has_high_fsc_at_limit = false; - has_large_incr_size_iter_ago = 0; - do_initialise_bodies = false; - - // By default, start with nr_bodies to 1 - mymodel.nr_bodies = 1; - fn_body_masks = "None"; - - // Debugging/analysis/hidden stuff - do_map = !checkParameter(argc, argv, "--no_map"); - minres_map = textToInteger(getParameter(argc, argv, "--minres_map", "5")); - abort_at_resolution = textToFloat(parser.getOption("--abort_at_resolution", "Abort when resolution reaches beyond this value", "-1", true)); - do_bfactor = checkParameter(argc, argv, "--bfactor"); - gridding_nr_iter = textToInteger(getParameter(argc, argv, "--gridding_iter", "10")); - debug1 = textToFloat(getParameter(argc, argv, "--debug1", "0")); - debug2 = textToFloat(getParameter(argc, argv, "--debug2", "0")); - debug3 = textToFloat(getParameter(argc, argv, "--debug3", "0")); - // Read in initial sigmaNoise spectrum - fn_sigma = getParameter(argc, argv, "--sigma",""); - do_calculate_initial_sigma_noise = (fn_sigma == "") ? true : false; - sigma2_fudge = textToFloat(getParameter(argc, argv, "--sigma2_fudge", "1")); - do_acc_currentsize_despite_highres_exp = checkParameter(argc, argv, "--accuracy_current_size"); - do_sequential_halves_recons = checkParameter(argc, argv, "--sequential_halves_recons"); - do_always_join_random_halves = checkParameter(argc, argv, "--always_join_random_halves"); - do_use_all_data = checkParameter(argc, argv, "--use_all_data"); - do_only_sample_tilt = checkParameter(argc, argv, "--only_sample_tilt"); - minimum_angular_sampling = textToFloat(getParameter(argc, argv, "--minimum_angular_sampling", "0")); - maximum_angular_sampling = textToFloat(getParameter(argc, argv, "--maximum_angular_sampling", "0")); - asymmetric_padding = parser.checkOption("--asymmetric_padding", "", "false", true); - skip_gridding = !parser.checkOption("--dont_skip_gridding", "Perform gridding in the reconstruction step (obsolete?)"); - debug_split_random_half = textToInteger(getParameter(argc, argv, "--debug_split_random_half", "0")); + { + std::cerr << "+ WARNING : Relion was compiled without CUDA of at least version 7.0 - you do NOT have support for GPUs" << std::endl; + do_gpu = false; + } +#endif + double temp_reqSize = textToDouble(parser.getOption("--free_gpu_memory", "GPU device memory (in Mb) to leave free after allocation.", "0")); + if(!do_zero_mask) + temp_reqSize += 100; + temp_reqSize *= 1000*1000; + if(temp_reqSize<0) + REPORT_ERROR("Invalid free_gpu_memory value."); + else + requested_free_gpu_memory = temp_reqSize; + + // Expert options + int expert_section = parser.addSection("Expert options"); + mymodel.padding_factor = textToFloat(parser.getOption("--pad", "Oversampling factor for the Fourier transforms of the references", "2")); + + ref_angpix = textToFloat(parser.getOption("--ref_angpix", "Pixel size (in A) for the input reference (default is to read from header)", "-1.")); + mymodel.interpolator = (parser.checkOption("--NN", "Perform nearest-neighbour instead of linear Fourier-space interpolation?")) ? NEAREST_NEIGHBOUR : TRILINEAR; + mymodel.r_min_nn = textToInteger(parser.getOption("--r_min_nn", "Minimum number of Fourier shells to perform linear Fourier-space interpolation", "10")); + verb = textToInteger(parser.getOption("--verb", "Verbosity (1=normal, 0=silent)", "1")); + random_seed = textToInteger(parser.getOption("--random_seed", "Number for the random seed generator", "-1")); + max_coarse_size = textToInteger(parser.getOption("--coarse_size", "Maximum image size for the first pass of the adaptive sampling approach", "-1")); + adaptive_fraction = textToFloat(parser.getOption("--adaptive_fraction", "Fraction of the weights to be considered in the first pass of adaptive oversampling ", "0.999")); + width_mask_edge = textToInteger(parser.getOption("--maskedge", "Width of the soft edge of the spherical mask (in pixels)", "5")); + // If we're doing helical, and maskedge is not given, use a default maskedge of 10 + if (helical_tube_outer_diameter > 0. && !checkParameter(argc, argv, "--maskedge")) width_mask_edge = 10.; + fix_sigma_noise = parser.checkOption("--fix_sigma_noise", "Fix the experimental noise spectra?"); + fix_sigma_offset = parser.checkOption("--fix_sigma_offset", "Fix the stddev in the origin offsets?"); + incr_size = textToInteger(parser.getOption("--incr_size", "Number of Fourier shells beyond the current resolution to be included in refinement", "10")); + do_print_metadata_labels = parser.checkOption("--print_metadata_labels", "Print a table with definitions of all metadata labels, and exit"); + do_print_symmetry_ops = parser.checkOption("--print_symmetry_ops", "Print all symmetry transformation matrices, and exit"); + strict_highres_exp = textToFloat(parser.getOption("--strict_highres_exp", "High resolution limit (in Angstrom) to restrict probability calculations in the expectation step", "-1")); + strict_lowres_exp = textToFloat(parser.getOption("--strict_lowres_exp", "Low resolution limit (in Angstrom) to restrict probability calculations in the expectation step", "-1")); + dont_raise_norm_error = parser.checkOption("--dont_check_norm", "Skip the check whether the images are normalised correctly"); + do_always_cc = parser.checkOption("--always_cc", "Perform CC-calculation in all iterations (useful for faster denovo model generation?)"); + do_phase_random_fsc = parser.checkOption("--solvent_correct_fsc", "Correct FSC curve for the effects of the solvent mask?"); + do_skip_maximization = parser.checkOption("--skip_maximize", "Skip maximization step (only write out data.star file)?"); + failsafe_threshold = textToInteger(parser.getOption("--failsafe_threshold", "Maximum number of particles permitted to be handled by fail-safe mode, due to zero sum of weights, before exiting with an error (GPU only).", "40")); + do_external_reconstruct = parser.checkOption("--external_reconstruct", "Perform the reconstruction step outside relion_refine, e.g. for learned priors?)"); + nr_iter_max = textToInteger(parser.getOption("--auto_iter_max", "In auto-refinement, stop at this iteration.", "999")); + auto_ignore_angle_changes = parser.checkOption("--auto_ignore_angles", "In auto-refinement, update angular sampling regardless of changes in orientations for convergence. This makes convergence faster."); + auto_resolution_based_angles= parser.checkOption("--auto_resol_angles", "In auto-refinement, update angular sampling based on resolution-based required sampling. This makes convergence faster."); + allow_coarser_samplings = parser.checkOption("--allow_coarser_sampling", "In 2D/3D classification, allow coarser angular and translational samplings if accuracies are bad (typically in earlier iterations."); + do_trust_ref_size = parser.checkOption("--trust_ref_size", "Trust the pixel and box size of the input reference; by default the program will die if these are different from the first optics group of the data"); + minimum_nr_particles_sigma2_noise = textToInteger(parser.getOption("--nr_parts_sigma2noise", "Number of particles (per optics group) for initial noise spectra estimation.", "1000")); + ///////////////// Special stuff for first iteration (only accessible via CL, not through readSTAR //////////////////// + + // When reading from the CL: always start at iteration 1 and subset 1 + iter = 0; + // When starting from CL: always calculate initial sigma_noise + do_calculate_initial_sigma_noise = true; + // Start average norm correction at 1! + mymodel.avg_norm_correction = 1.; + // Always initialise the PDF of the directions + directions_have_changed = true; + + // Only reconstruct and join random halves are only available when continuing an old run + do_join_random_halves = false; + + // For auto-sampling and convergence check + nr_iter_wo_resol_gain = 0; + nr_iter_wo_large_hidden_variable_changes = 0; + current_changes_optimal_classes = 9999999; + current_changes_optimal_offsets = 999.; + current_changes_optimal_orientations = 999.; + smallest_changes_optimal_classes = 9999999; + smallest_changes_optimal_offsets = 999.; + smallest_changes_optimal_orientations = 999.; + acc_rot = acc_trans = 999.; + + best_resol_thus_far = 1./999.; + has_converged = false; + has_high_fsc_at_limit = false; + has_large_incr_size_iter_ago = 0; + do_initialise_bodies = false; + + // By default, start with nr_bodies to 1 + mymodel.nr_bodies = 1; + fn_body_masks = "None"; + + // Debugging/analysis/hidden stuff + do_map = !checkParameter(argc, argv, "--no_map"); + minres_map = textToInteger(getParameter(argc, argv, "--minres_map", "5")); + abort_at_resolution = textToFloat(parser.getOption("--abort_at_resolution", "Abort when resolution reaches beyond this value", "-1", true)); + do_bfactor = checkParameter(argc, argv, "--bfactor"); + gridding_nr_iter = textToInteger(getParameter(argc, argv, "--gridding_iter", "10")); + debug1 = textToFloat(getParameter(argc, argv, "--debug1", "0")); + debug2 = textToFloat(getParameter(argc, argv, "--debug2", "0")); + debug3 = textToFloat(getParameter(argc, argv, "--debug3", "0")); + // Read in initial sigmaNoise spectrum + fn_sigma = getParameter(argc, argv, "--sigma",""); + do_calculate_initial_sigma_noise = (fn_sigma == "") ? true : false; + sigma2_fudge = textToFloat(getParameter(argc, argv, "--sigma2_fudge", "1")); + do_acc_currentsize_despite_highres_exp = checkParameter(argc, argv, "--accuracy_current_size"); + do_sequential_halves_recons = checkParameter(argc, argv, "--sequential_halves_recons"); + do_always_join_random_halves = checkParameter(argc, argv, "--always_join_random_halves"); + do_use_all_data = checkParameter(argc, argv, "--use_all_data"); + do_only_sample_tilt = checkParameter(argc, argv, "--only_sample_tilt"); + minimum_angular_sampling = textToFloat(getParameter(argc, argv, "--minimum_angular_sampling", "0")); + maximum_angular_sampling = textToFloat(getParameter(argc, argv, "--maximum_angular_sampling", "0")); + asymmetric_padding = parser.checkOption("--asymmetric_padding", "", "false", true); + skip_gridding = !parser.checkOption("--dont_skip_gridding", "Perform gridding in the reconstruction step (obsolete?)"); + debug_split_random_half = textToInteger(getParameter(argc, argv, "--debug_split_random_half", "0")); skip_realspace_helical_sym = parser.checkOption("--skip_realspace_helical_sym", "", "false", true); #ifdef DEBUG_READ - std::cerr<<"MlOptimiser::parseInitial Done"<