From 0f4d08df0d77fa84b74e0a7a7c075e6b24918ad3 Mon Sep 17 00:00:00 2001 From: jmelou Date: Fri, 21 Jul 2023 18:33:15 +0200 Subject: [PATCH 01/67] [mvsUtils] Add error case --- src/aliceVision/mvsUtils/fileIO.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/aliceVision/mvsUtils/fileIO.cpp b/src/aliceVision/mvsUtils/fileIO.cpp index c641ed12b8..8223310c9a 100644 --- a/src/aliceVision/mvsUtils/fileIO.cpp +++ b/src/aliceVision/mvsUtils/fileIO.cpp @@ -336,6 +336,10 @@ std::string getFileNameFromViewId(const MultiViewParams& mp, ext = "obj"; break; } + case EFileType::none: + { + ALICEVISION_THROW_ERROR("FileType is None"); + } } const std::string fileName = folder + std::to_string(viewId) + suffix + customSuffix + tileSuffix + "." + ext; From 3b2df20af00c4fcfc14bb6513cdb96255994ad97 Mon Sep 17 00:00:00 2001 From: jmelou Date: Fri, 21 Jul 2023 18:34:32 +0200 Subject: [PATCH 02/67] [PS] Add check for metadata --- .../photometricStereo/photometricStereo.cpp | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/src/aliceVision/photometricStereo/photometricStereo.cpp b/src/aliceVision/photometricStereo/photometricStereo.cpp index 59b4b82b4f..da3dbb186a 100644 --- a/src/aliceVision/photometricStereo/photometricStereo.cpp +++ b/src/aliceVision/photometricStereo/photometricStereo.cpp @@ -112,17 +112,29 @@ void photometricStereo(const sfmData::SfMData& sfmData, ALICEVISION_LOG_INFO("Pose Id: " << posesIt.first); std::vector& initViewIds = posesIt.second; - std::map idMap; - for (auto& viewId : initViewIds) - { - std::map currentMetadata = sfmData.getView(viewId).getImage().getMetadata(); - idMap[currentMetadata.at("Exif:DateTimeDigitized")] = viewId; - } + bool hasMetadata = true; + std::map metadataTest = sfmData.getView(initViewIds.at(0)).getImage().getMetadata(); + if (metadataTest.find("Exif:DateTimeDigitized") == metadataTest.end()) + hasMetadata = false; std::vector viewIds; - for (const auto& [currentTime, viewId] : idMap) + if (hasMetadata) + { + std::map idMap; + for (auto& viewId : initViewIds) + { + std::map currentMetadata = sfmData.getView(viewId).getImage().getMetadata(); + idMap[currentMetadata.at("Exif:DateTimeDigitized")] = viewId; + } + + for (const auto& [currentTime, viewId] : idMap) + { + viewIds.push_back(viewId); + } + } + else { - viewIds.push_back(viewId); + viewIds = initViewIds; } for (auto& viewId : viewIds) From 7100998b1fe2cf733471c392c98265e90309af1a Mon Sep 17 00:00:00 2001 From: jmelou Date: Fri, 21 Jul 2023 18:36:07 +0200 Subject: [PATCH 03/67] [RTI] Add import from Lp file (RTI) --- .../photometricStereo/photometricDataIO.cpp | 51 +++++++++++++++++++ .../photometricStereo/photometricDataIO.hpp | 5 ++ .../photometricStereo/photometricStereo.cpp | 11 +++- 3 files changed, 66 insertions(+), 1 deletion(-) diff --git a/src/aliceVision/photometricStereo/photometricDataIO.cpp b/src/aliceVision/photometricStereo/photometricDataIO.cpp index 174337eb35..f86e1c3af2 100644 --- a/src/aliceVision/photometricStereo/photometricDataIO.cpp +++ b/src/aliceVision/photometricStereo/photometricDataIO.cpp @@ -241,6 +241,57 @@ void buildLightMatFromJSON(const std::string& fileName, } } +void buildLightMatFromLP(const std::string& fileName, + const std::vector& imageList, + Eigen::MatrixXf& lightMat, + std::vector>& intList) +{ + std::string pictureName; + float x, y, z; + + int lineNumber = 0; + std::array intensities = {1.0, 1.0, 1.0}; + + for (auto& currentImPath : imageList) + { + std::stringstream stream; + std::string line; + std::fstream intFile; + intFile.open(fileName, std::ios::in); + + if (!intFile.is_open()) + { + ALICEVISION_LOG_ERROR("Unable to load Lp file"); + ALICEVISION_THROW_ERROR("Cannot open '" << fileName << "'!"); + } + else + { + fs::path imagePathFS = fs::path(currentImPath); + while (!intFile.eof()) + { + std::getline(intFile, line); + stream.clear(); + stream.str(line); + + stream >> pictureName >> x >> y >> z; + + std::string stringToCompare = imagePathFS.filename().string(); + + if (boost::algorithm::iequals(pictureName, stringToCompare)) + { + lightMat(lineNumber, 0) = x; + lightMat(lineNumber, 1) = -y; + lightMat(lineNumber, 2) = -z; + ++lineNumber; + + intList.push_back(intensities); + } + } + intFile.close(); + } + } +} + void loadMask(std::string const& maskName, image::Image& mask) { if (maskName.empty() || !utils::exists(maskName)) diff --git a/src/aliceVision/photometricStereo/photometricDataIO.hpp b/src/aliceVision/photometricStereo/photometricDataIO.hpp index e1e71bc4fc..02d2bdc1cf 100644 --- a/src/aliceVision/photometricStereo/photometricDataIO.hpp +++ b/src/aliceVision/photometricStereo/photometricDataIO.hpp @@ -66,6 +66,11 @@ void buildLightMatFromJSON(const std::string& fileName, Eigen::MatrixXf& lightMat, std::vector>& intList); +void buildLightMatFromLP(const std::string& fileName, + const std::vector& imageList, + Eigen::MatrixXf& lightMat, + std::vector>& intList); + /** * @brief Load a mask * @param[in] maskName Path to mask diff --git a/src/aliceVision/photometricStereo/photometricStereo.cpp b/src/aliceVision/photometricStereo/photometricStereo.cpp index da3dbb186a..c3b88f6aa9 100644 --- a/src/aliceVision/photometricStereo/photometricStereo.cpp +++ b/src/aliceVision/photometricStereo/photometricStereo.cpp @@ -160,7 +160,16 @@ void photometricStereo(const sfmData::SfMData& sfmData, } else { - buildLightMatFromJSON(lightData, viewIds, lightMat, intList); + const std::string extension = fs::path(lightData).extension(); + + if (extension == ".json") // JSON File + { + buildLightMatFromJSON(lightData, viewIds, lightMat, intList); + } + else if (extension == ".lp") + { + buildLightMatFromLP(lightData, imageList, lightMat, intList); + } } /* Ensure that there are as many images as light calibrations, and that the list of input images is not empty. From 6634a7a8df62cc5191b565d5af78c58d3b48933e Mon Sep 17 00:00:00 2001 From: jmelou Date: Fri, 21 Jul 2023 18:37:08 +0200 Subject: [PATCH 04/67] [PS] Add some breaks --- src/aliceVision/photometricStereo/photometricDataIO.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/aliceVision/photometricStereo/photometricDataIO.cpp b/src/aliceVision/photometricStereo/photometricDataIO.cpp index f86e1c3af2..4da8f8a262 100644 --- a/src/aliceVision/photometricStereo/photometricDataIO.cpp +++ b/src/aliceVision/photometricStereo/photometricDataIO.cpp @@ -213,6 +213,7 @@ void buildLightMatFromJSON(const std::string& fileName, lightMat(lightIndex, cpt) = direction.second.get_value(); ++cpt; } + break; } else { @@ -235,6 +236,7 @@ void buildLightMatFromJSON(const std::string& fileName, ++cpt; } ++lineNumber; + break; } } } From 8fcbc90565cd609342781e596783797d1bdc105d Mon Sep 17 00:00:00 2001 From: jmelou Date: Fri, 21 Jul 2023 18:38:19 +0200 Subject: [PATCH 05/67] [lightingCalibration] Use perspective hypothesis for brightest point method --- .../lightingEstimation/lightingCalibration.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/aliceVision/lightingEstimation/lightingCalibration.cpp b/src/aliceVision/lightingEstimation/lightingCalibration.cpp index 4b8ca502c3..4c3f6f681e 100644 --- a/src/aliceVision/lightingEstimation/lightingCalibration.cpp +++ b/src/aliceVision/lightingEstimation/lightingCalibration.cpp @@ -132,15 +132,15 @@ void lightCalibrationOneImage(const std::string& picturePath, getNormalOnSphere(brigthestPoint(0), brigthestPoint(1), sphereParam, normalBrightestPoint); // Observation direction : - Eigen::Vector3f observationRay; + Eigen::Vector3f observationRayPersp; - // orthographic approximation : - observationRay(0) = 0.0; - observationRay(1) = 0.0; - observationRay(2) = -1.0; + observationRayPersp(0) = (brigthestPoint(0) - imageFloat.cols() / 2) / focal; + observationRayPersp(1) = (brigthestPoint(1) - imageFloat.rows() / 2) / focal; + observationRayPersp(2) = 1; + observationRayPersp = -observationRayPersp / observationRayPersp.norm(); // Evaluate lighting direction : - lightingDirection = 2 * normalBrightestPoint.dot(observationRay) * normalBrightestPoint - observationRay; + lightingDirection = 2 * normalBrightestPoint.dot(observationRayPersp) * normalBrightestPoint - observationRayPersp; lightingDirection = lightingDirection / lightingDirection.norm(); } // If method = HS : From 013aebb8d401fe123308998c998c97fc72042bfa Mon Sep 17 00:00:00 2001 From: jmelou Date: Thu, 7 Sep 2023 15:26:49 +0200 Subject: [PATCH 06/67] [PS] Typo and break for Lp files --- src/aliceVision/photometricStereo/photometricDataIO.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/aliceVision/photometricStereo/photometricDataIO.cpp b/src/aliceVision/photometricStereo/photometricDataIO.cpp index 4da8f8a262..2b0221ae1e 100644 --- a/src/aliceVision/photometricStereo/photometricDataIO.cpp +++ b/src/aliceVision/photometricStereo/photometricDataIO.cpp @@ -207,7 +207,6 @@ void buildLightMatFromJSON(const std::string& fileName, intList.push_back(currentIntensities); cpt = 0; - for (auto& direction : light.second.get_child("direction")) { lightMat(lightIndex, cpt) = direction.second.get_value(); @@ -287,6 +286,8 @@ void buildLightMatFromLP(const std::string& fileName, ++lineNumber; intList.push_back(intensities); + + break; } } intFile.close(); From 5ce4f3600563ded4945cfb551675162ed65d8d2f Mon Sep 17 00:00:00 2001 From: jmelou Date: Tue, 12 Sep 2023 17:20:24 +0200 Subject: [PATCH 07/67] [texturing] Add default filetype (none) --- src/aliceVision/mvsUtils/MultiViewParams.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/aliceVision/mvsUtils/MultiViewParams.hpp b/src/aliceVision/mvsUtils/MultiViewParams.hpp index 404ed430eb..d82461d904 100644 --- a/src/aliceVision/mvsUtils/MultiViewParams.hpp +++ b/src/aliceVision/mvsUtils/MultiViewParams.hpp @@ -80,6 +80,7 @@ enum class EFileType volumeTopographicCut = 50, stats9p = 51, tilePattern = 52, + none = 9999 }; class MultiViewParams From 4f8d343266004e6228fa95057d08fcaf5789b65b Mon Sep 17 00:00:00 2001 From: jmelou Date: Tue, 12 Sep 2023 17:21:54 +0200 Subject: [PATCH 08/67] [multi_view] Change MultiViewParams constructor --- src/aliceVision/mvsUtils/MultiViewParams.cpp | 9 +++++++-- src/aliceVision/mvsUtils/MultiViewParams.hpp | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/aliceVision/mvsUtils/MultiViewParams.cpp b/src/aliceVision/mvsUtils/MultiViewParams.cpp index 98cf1d4557..2cb0ad67ad 100644 --- a/src/aliceVision/mvsUtils/MultiViewParams.cpp +++ b/src/aliceVision/mvsUtils/MultiViewParams.cpp @@ -38,7 +38,7 @@ MultiViewParams::MultiViewParams(const sfmData::SfMData& sfmData, const std::string& imagesFolder, const std::string& depthMapsFolder, const std::string& depthMapsFilterFolder, - bool readFromDepthMaps, + mvsUtils::EFileType fileType, int downscale) : _sfmData(sfmData), _imagesFolder(imagesFolder + "/"), @@ -63,7 +63,7 @@ MultiViewParams::MultiViewParams(const sfmData::SfMData& sfmData, std::string path = view.getImage().getImagePath(); - if (readFromDepthMaps) + if (fileType == mvsUtils::EFileType::depthMap) { if (depthMapsFolder.empty()) { @@ -76,6 +76,11 @@ MultiViewParams::MultiViewParams(const sfmData::SfMData& sfmData, path = getFileNameFromViewId(*this, view.getViewId(), mvsUtils::EFileType::depthMap); } } + else if (fileType == mvsUtils::EFileType::normalMap) + { + const int scale = 0; + path = getFileNameFromViewId(*this, view.getViewId(), mvsUtils::EFileType::normalMap, scale); + } else if (_imagesFolder != "/" && !_imagesFolder.empty() && fs::is_directory(_imagesFolder) && !fs::is_empty(_imagesFolder)) { // find folder file extension diff --git a/src/aliceVision/mvsUtils/MultiViewParams.hpp b/src/aliceVision/mvsUtils/MultiViewParams.hpp index d82461d904..04648a0331 100644 --- a/src/aliceVision/mvsUtils/MultiViewParams.hpp +++ b/src/aliceVision/mvsUtils/MultiViewParams.hpp @@ -117,7 +117,7 @@ class MultiViewParams const std::string& imagesFolder = "", const std::string& depthMapsFolder = "", const std::string& depthMapsFilterFolder = "", - bool readFromDepthMaps = false, + mvsUtils::EFileType fileType = mvsUtils::EFileType::none, int downscale = 1); ~MultiViewParams(); From 8cbd151ebd2bee9c2ddbaea08f639dd740aa519e Mon Sep 17 00:00:00 2001 From: jmelou Date: Tue, 12 Sep 2023 17:23:29 +0200 Subject: [PATCH 09/67] [multi-view] Change call to MultiViewParams constructor --- src/aliceVision/fuseCut/DelaunayGraphCut_test.cpp | 2 +- src/software/pipeline/main_depthMapEstimation.cpp | 2 +- src/software/pipeline/main_depthMapFiltering.cpp | 2 +- src/software/utils/main_lightingEstimation.cpp | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/aliceVision/fuseCut/DelaunayGraphCut_test.cpp b/src/aliceVision/fuseCut/DelaunayGraphCut_test.cpp index 58c1e110f2..5ce366cccd 100644 --- a/src/aliceVision/fuseCut/DelaunayGraphCut_test.cpp +++ b/src/aliceVision/fuseCut/DelaunayGraphCut_test.cpp @@ -67,7 +67,7 @@ BOOST_AUTO_TEST_CASE(fuseCut_delaunayGraphCut) const NViewDatasetConfigurator config(1000, 1000, 500, 500, 1, 0); SfMData sfmData = generateSfm(config, 6); - mvsUtils::MultiViewParams mp(sfmData, "", "", "", false); + mvsUtils::MultiViewParams mp(sfmData, "", "", ""); mp.userParams.put("LargeScale.universePercentile", 0.999); mp.userParams.put("delaunaycut.seed", 1); diff --git a/src/software/pipeline/main_depthMapEstimation.cpp b/src/software/pipeline/main_depthMapEstimation.cpp index 4d46a50781..fb47c71243 100644 --- a/src/software/pipeline/main_depthMapEstimation.cpp +++ b/src/software/pipeline/main_depthMapEstimation.cpp @@ -296,7 +296,7 @@ int aliceVision_main(int argc, char* argv[]) } // MultiViewParams initialization - mvsUtils::MultiViewParams mp(sfmData, imagesFolder, outputFolder, "", false, downscale); + mvsUtils::MultiViewParams mp(sfmData, imagesFolder, outputFolder, "", mvsUtils::EFileType::none, downscale); // set MultiViewParams min/max view angle mp.setMinViewAngle(minViewAngle); diff --git a/src/software/pipeline/main_depthMapFiltering.cpp b/src/software/pipeline/main_depthMapFiltering.cpp index 56b61c3ad9..17c98ce92e 100644 --- a/src/software/pipeline/main_depthMapFiltering.cpp +++ b/src/software/pipeline/main_depthMapFiltering.cpp @@ -107,7 +107,7 @@ int aliceVision_main(int argc, char* argv[]) } // initialization - mvsUtils::MultiViewParams mp(sfmData, "", depthMapsFolder, outputFolder, "", true); + mvsUtils::MultiViewParams mp(sfmData, "", depthMapsFolder, outputFolder, mvsUtils::EFileType::depthMap); mp.setMinViewAngle(minViewAngle); mp.setMaxViewAngle(maxViewAngle); diff --git a/src/software/utils/main_lightingEstimation.cpp b/src/software/utils/main_lightingEstimation.cpp index ac2ea50fbc..6e11c0b4f3 100644 --- a/src/software/utils/main_lightingEstimation.cpp +++ b/src/software/utils/main_lightingEstimation.cpp @@ -331,7 +331,7 @@ int main(int argc, char** argv) } // initialization - mvsUtils::MultiViewParams mp(sfmData, imagesFolder, "", depthMapsFilterFolder, false); + mvsUtils::MultiViewParams mp(sfmData, imagesFolder, "", depthMapsFilterFolder, mvsUtils::EFileType::none); lightingEstimation::LighthingEstimator estimator; From cc17359c275d2ef50953e2d1ec895136be63e07a Mon Sep 17 00:00:00 2001 From: jmelou Date: Tue, 12 Sep 2023 17:24:30 +0200 Subject: [PATCH 10/67] [texturing] Add margin as parameter --- src/aliceVision/mesh/Texturing.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/aliceVision/mesh/Texturing.cpp b/src/aliceVision/mesh/Texturing.cpp index 315674ff92..c0998d4861 100644 --- a/src/aliceVision/mesh/Texturing.cpp +++ b/src/aliceVision/mesh/Texturing.cpp @@ -140,7 +140,7 @@ std::istream& operator>>(std::istream& in, EBumpMappingType& bumpMappingType) * coordinates of this pixel relative to \p triangle * @return */ -bool isPixelInTriangle(const Point2d* triangle, const Pixel& pixel, Point2d& barycentricCoords) +bool isPixelInTriangle(const Point2d* triangle, const Pixel& pixel, Point2d& barycentricCoords, double margin = 0.5) { // get pixel center GEO::vec2 p(pixel.x + 0.5, pixel.y + 0.5); @@ -154,7 +154,7 @@ bool isPixelInTriangle(const Point2d* triangle, const Pixel& pixel, Point2d& bar barycentricCoords.x = l3; barycentricCoords.y = l2; // tolerance threshold of 1/2 pixel for pixels on the edges of the triangle - return dist < 0.5 + std::numeric_limits::epsilon(); + return dist < margin + std::numeric_limits::epsilon(); } Point2d barycentricToCartesian(const Point2d* triangle, const Point2d& coords) From 4da0075b9aa66675192ceea1a2c4895a2dc035ab Mon Sep 17 00:00:00 2001 From: jmelou Date: Tue, 12 Sep 2023 17:30:25 +0200 Subject: [PATCH 11/67] [texturing] Allow texturing with normalMaps --- src/aliceVision/mesh/Texturing.cpp | 394 +++++++++++++------ src/aliceVision/mesh/Texturing.hpp | 9 +- src/aliceVision/mvsUtils/MultiViewParams.cpp | 3 +- src/software/pipeline/main_meshing.cpp | 2 +- src/software/pipeline/main_texturing.cpp | 25 +- 5 files changed, 295 insertions(+), 138 deletions(-) diff --git a/src/aliceVision/mesh/Texturing.cpp b/src/aliceVision/mesh/Texturing.cpp index c0998d4861..fb55dcaa70 100644 --- a/src/aliceVision/mesh/Texturing.cpp +++ b/src/aliceVision/mesh/Texturing.cpp @@ -167,6 +167,123 @@ Point3d barycentricToCartesian(const Point3d* triangle, const Point2d& coords) return triangle[0] + (triangle[2] - triangle[0]) * coords.x + (triangle[1] - triangle[0]) * coords.y; } +inline GEO::vec3 mesh_facet_interpolate_normal_at_point(const GEO::Mesh& mesh, GEO::index_t f, const GEO::vec3& p) +{ + const GEO::index_t v0 = mesh.facets.vertex(f, 0); + const GEO::index_t v1 = mesh.facets.vertex(f, 1); + const GEO::index_t v2 = mesh.facets.vertex(f, 2); + + const GEO::vec3 p0 = mesh.vertices.point(v0); + const GEO::vec3 p1 = mesh.vertices.point(v1); + const GEO::vec3 p2 = mesh.vertices.point(v2); + + const GEO::vec3 n0 = GEO::normalize(GEO::Geom::mesh_vertex_normal(mesh, v0)); + const GEO::vec3 n1 = GEO::normalize(GEO::Geom::mesh_vertex_normal(mesh, v1)); + const GEO::vec3 n2 = GEO::normalize(GEO::Geom::mesh_vertex_normal(mesh, v2)); + + GEO::vec3 barycCoords; + GEO::vec3 closestPoint; + GEO::Geom::point_triangle_squared_distance(p, p0, p1, p2, closestPoint, barycCoords.x, barycCoords.y, barycCoords.z); + + const GEO::vec3 n = barycCoords.x * n0 + barycCoords.y * n1 + barycCoords.z * n2; + + return GEO::normalize(n); +} + +inline GEO::vec3 mesh_facet_interpolate_normal_at_point(const StaticVector& ptsNormals, const Mesh& mesh, GEO::index_t f, const GEO::vec3& p) +{ + const GEO::index_t v0 = (mesh.tris)[f].v[0]; + const GEO::index_t v1 = (mesh.tris)[f].v[1]; + const GEO::index_t v2 = (mesh.tris)[f].v[2]; + + const GEO::vec3 p0((mesh.pts)[v0].x, (mesh.pts)[v0].y, (mesh.pts)[v0].z); + const GEO::vec3 p1((mesh.pts)[v1].x, (mesh.pts)[v1].y, (mesh.pts)[v1].z); + const GEO::vec3 p2((mesh.pts)[v2].x, (mesh.pts)[v2].y, (mesh.pts)[v2].z); + + const GEO::vec3 n0(ptsNormals[v0].x, ptsNormals[v0].y, ptsNormals[v0].z); + const GEO::vec3 n1(ptsNormals[v1].x, ptsNormals[v1].y, ptsNormals[v1].z); + const GEO::vec3 n2(ptsNormals[v2].x, ptsNormals[v2].y, ptsNormals[v2].z); + + GEO::vec3 barycCoords; + GEO::vec3 closestPoint; + GEO::Geom::point_triangle_squared_distance(p, p0, p1, p2, closestPoint, barycCoords.x, barycCoords.y, barycCoords.z); + + const GEO::vec3 n = barycCoords.x * n0 + barycCoords.y * n1 + barycCoords.z * n2; + + return GEO::normalize(n); +} + +template +inline Eigen::Matrix toEigen(const GEO::vecng& v) +{ + return Eigen::Matrix(v.data()); +} + +/** + * @brief Compute a transformation matrix to convert coordinates in world space coordinates into the triangle space. + * The triangle space is define by the Z-axis as the normal of the triangle, + * the X-axis aligned with the horizontal line in the texture file (using texture/UV coordinates). + * + * @param[in] mesh: input mesh + * @param[in] f: facet/triangle index + * @param[in] triPts: UV Coordinates + * @return Rotation matrix to convert from world space coordinates in the triangle space + */ +inline Eigen::Matrix3d computeTriangleTransform(const Mesh& mesh, int f, const Point2d* triPts) +{ + const Eigen::Vector3d p0 = toEigen((mesh.pts)[(mesh.tris)[f].v[0]]); + const Eigen::Vector3d p1 = toEigen((mesh.pts)[(mesh.tris)[f].v[1]]); + const Eigen::Vector3d p2 = toEigen((mesh.pts)[(mesh.tris)[f].v[2]]); + + const Eigen::Vector3d tX = (p1 - p0).normalized(); // edge0 => local triangle X-axis + const Eigen::Vector3d N = tX.cross((p2 - p0).normalized()).normalized(); // cross(edge0, edge1) => Z-axis + + // Correct triangle X-axis to be align with X-axis in the texture + const GEO::vec2 t0 = GEO::vec2(triPts[0].m); + const GEO::vec2 t1 = GEO::vec2(triPts[1].m); + const GEO::vec2 tV = GEO::normalize(t1 - t0); + const GEO::vec2 origNormal(1.0, 0.0); // X-axis in the texture + const double tAngle = GEO::Geom::angle(tV, origNormal); + Eigen::Matrix3d transform(Eigen::AngleAxisd(tAngle, N).toRotationMatrix()); + // Rotate triangle v0v1 axis around Z-axis, to get a X axis aligned with the 2d texture + Eigen::Vector3d X = (transform * tX).normalized(); + + const Eigen::Vector3d Y = N.cross(X).normalized(); // Y-axis + + Eigen::Matrix3d m; + m.col(0) = X; + m.col(1) = Y; + m.col(2) = N; + // const Eigen::Matrix3d mInv = m.inverse(); + const Eigen::Matrix3d mT = m.transpose(); + + return mT; +} + +inline void computeNormalHeight(const GEO::Mesh& mesh, + double orientation, + double t, + GEO::index_t f, + const Eigen::Matrix3d& m, + const GEO::vec3& q, + const GEO::vec3& qA, + const GEO::vec3& qB, + float& out_height, + image::RGBfColor& out_normal) +{ + GEO::vec3 intersectionPoint = t * qB + (1.0 - t) * qA; + out_height = q.distance(intersectionPoint) * orientation; + + // Use facet normal + // GEO::vec3 denseMeshNormal_f = normalize(GEO::Geom::mesh_facet_normal(mesh, f)); + // Use per pixel normal using weighted interpolation of the facet vertex normals + const GEO::vec3 denseMeshNormal = mesh_facet_interpolate_normal_at_point(mesh, f, intersectionPoint); + + Eigen::Vector3d dNormal = m * toEigen(denseMeshNormal); + dNormal.normalize(); + out_normal = image::RGBfColor(dNormal(0), dNormal(1), dNormal(2)); +} + void Texturing::generateUVsBasicMethod(mvsUtils::MultiViewParams& mp) { if (!mesh) @@ -282,7 +399,8 @@ void Texturing::updateAtlases() void Texturing::generateTextures(const mvsUtils::MultiViewParams& mp, const fs::path& outPath, size_t memoryAvailable, - image::EImageFileType textureFileType) + image::EImageFileType textureFileType, + mvsUtils::EFileType imageType) { // Ensure that contribution levels do not contain 0 and are sorted (as each frequency band contributes to lower bands). auto& m = texParams.multiBandNbContrib; @@ -363,7 +481,7 @@ void Texturing::generateTextures(const mvsUtils::MultiViewParams& mp, atlasIDs.push_back(atlasID); } ALICEVISION_LOG_INFO("Generating texture for atlases " << n * nbAtlasMax + 1 << " to " << n * nbAtlasMax + imax); - generateTexturesSubSet(mp, atlasIDs, imageCache, outPath, textureFileType); + generateTexturesSubSet(mp, atlasIDs, imageCache, outPath, textureFileType, imageType); } } @@ -371,7 +489,8 @@ void Texturing::generateTexturesSubSet(const mvsUtils::MultiViewParams& mp, const std::vector& atlasIDs, mvsUtils::ImagesCache>& imageCache, const fs::path& outPath, - image::EImageFileType textureFileType) + image::EImageFileType textureFileType, + mvsUtils::EFileType imageType) { if (atlasIDs.size() > _atlases.size()) throw std::runtime_error("Invalid atlas IDs "); @@ -740,7 +859,126 @@ void Texturing::generateTexturesSubSet(const mvsUtils::MultiViewParams& mp, } } } - writeTexture(atlasTexture, atlasID, outPath, textureFileType, -1); + + // If mode "normalMaps" + if (imageType == mvsUtils::EFileType::normalMap) + { + // Rotation and normalization of normals +#pragma omp parallel for + for (size_t i = 0; i < _atlases[atlasID].size(); ++i) + { + // Eigen::Matrix visitedPixels(atlasTexture.img.rows(), atlasTexture.img.cols()); + // visitedPixels.fill(false); + int triangleId = _atlases[atlasID][i]; + + // Retrieve triangle 3D and UV coordinates + Point2d triPixs[3]; + Point3d triPts[3]; + auto& triangleUvIds = mesh->trisUvIds[triangleId]; + + // Retrieve triangle normal + + // Compute the Bottom-Left minima of the current UDIM for [0,1] range remapping + Point2d udimBL; + StaticVector& uvCoords = mesh->uvCoords; + udimBL.x = std::floor(std::min({uvCoords[triangleUvIds[0]].x, uvCoords[triangleUvIds[1]].x, uvCoords[triangleUvIds[2]].x})); + udimBL.y = std::floor(std::min({uvCoords[triangleUvIds[0]].y, uvCoords[triangleUvIds[1]].y, uvCoords[triangleUvIds[2]].y})); + + for (int k = 0; k < 3; k++) + { + const int pointIndex = (mesh->tris)[triangleId].v[k]; + triPts[k] = (mesh->pts)[pointIndex]; // 3D coordinates + const int uvPointIndex = triangleUvIds.m[k]; + + Point2d uv = uvCoords[uvPointIndex]; + // UDIM: remap coordinates between [0,1] + uv = uv - udimBL; + + triPixs[k] = uv * texParams.textureSide; // UV coordinates + } + + // compute triangle bounding box in pixel indexes + // min values: floor(value) + // max values: ceil(value) + Pixel LU, RD; + LU.x = static_cast(std::floor(std::min({triPixs[0].x, triPixs[1].x, triPixs[2].x}))); + LU.y = static_cast(std::floor(std::min({triPixs[0].y, triPixs[1].y, triPixs[2].y}))); + RD.x = static_cast(std::ceil(std::max({triPixs[0].x, triPixs[1].x, triPixs[2].x}))); + RD.y = static_cast(std::ceil(std::max({triPixs[0].y, triPixs[1].y, triPixs[2].y}))); + + // sanity check: clamp values to [0; textureSide] + int texSide = static_cast(texParams.textureSide); + LU.x = clamp(LU.x, 0, texSide); + LU.y = clamp(LU.y, 0, texSide); + RD.x = clamp(RD.x, 0, texSide); + RD.y = clamp(RD.y, 0, texSide); + + const Eigen::Matrix3d worldToTriangleMatrix = computeTriangleTransform(*mesh, triangleId, triPixs); + + // iterate over bounding box's pixels + for (int y = LU.y; y < RD.y; ++y) + { + for (int x = LU.x; x < RD.x; ++x) + { + Pixel pix(x, y); // top-left corner of the pixel + Point2d barycCoords; + + // test if the pixel is inside triangle + // and retrieve its barycentric coordinates + const double margin = 0.5; + if (!isPixelInTriangle(triPixs, pix, barycCoords, margin)) + { + continue; + } + + // bool visited; + // #pragma omp critical + // { + // visited = visitedPixels(x,y); + // visitedPixels(x,y) = true; + // } + + // if(visited) + // { + // continue; + // } + + // remap 'y' to image coordinates system (inverted Y axis) + const unsigned int y_ = (texParams.textureSide - 1) - y; + // 1D pixel index + const unsigned int xyoffset = y_ * texParams.textureSide + x; + + /* + // get 3D coordinates + const Point3d pt3d = barycentricToCartesian(triPts, barycCoords); + const GEO::vec3 q(pt3d.x, pt3d.y, pt3d.z); + + // Texel normal (weighted normal from the 3 vertices normals), instead of face normal for better + // transitions (reduce seams) + const GEO::vec3 triangleNormal_p = mesh_facet_interpolate_normal_at_point(sparseMesh, triangleId, q); + // const GEO::vec3 triangleNormal_p = GEO::vec3(triangleNormal.m); // to use the triangle normal instead + const GEO::vec3 scaledTriangleNormal = triangleNormal_p * minEdgeLength * 10; // ?????? + + */ + + Vec3 origNormal = atlasTexture.img(xyoffset).cast(); + if (origNormal[2] > 0) + { + continue; + } + origNormal.normalize(); + origNormal = worldToTriangleMatrix * origNormal; + atlasTexture.img(xyoffset) = image::RGBfColor(origNormal[0], origNormal[1], origNormal[2]); + + // Normal in visual representation + /*normalMap(i) = image::RGBfColor(normalMap(i).r() * 0.5 + 0.5, + normalMap(i).g() * 0.5 + 0.5, + normalMap(i).b() * 0.5 + 0.5); // B: -1:+1 => 0-255 which means 0:+1 => 128-255*/ + } + } + } + } + writeTexture(atlasTexture, atlasID, outPath, textureFileType, -1, imageType); } } @@ -768,7 +1006,8 @@ void Texturing::writeTexture(AccuImage& atlasTexture, const std::size_t atlasID, const std::filesystem::path& outPath, image::EImageFileType textureFileType, - const int level) + const int level, + mvsUtils::EFileType imageType) { unsigned int outTextureSide = texParams.textureSide; // WARNING: we modify the "imgCount" to apply the padding (to avoid the creation of a new buffer) @@ -891,14 +1130,34 @@ void Texturing::writeTexture(AccuImage& atlasTexture, ALICEVISION_LOG_INFO(" - Downscaling texture (" << texParams.downscale << "x)."); imageAlgo::resizeImage(texParams.downscale, atlasTexture.img, resizedColorBuffer); + + // Normalize normal map after dowscaling + /*if(imageType == mvsUtils::EFileType::normalMap) + { + for(int i = 0; i < resizedColorBuffer.size(); ++i) + { + resizedColorBuffer(i).normalize(); + } + }*/ + std::swap(resizedColorBuffer, atlasTexture.img); } - material.diffuseType = textureFileType; - const std::string textureName = material.textureName(Material::TextureType::DIFFUSE, static_cast(atlasID)); - material.addTexture(Material::TextureType::DIFFUSE, textureName); + std::string textureName; + if (imageType == mvsUtils::EFileType::normalMap) + { + material.bumpType = textureFileType; + textureName = material.textureName(Material::TextureType::BUMP, static_cast(atlasID)); + material.addTexture(Material::TextureType::BUMP, textureName); + } + else + { + material.diffuseType = textureFileType; + textureName = material.textureName(Material::TextureType::DIFFUSE, static_cast(atlasID)); + material.addTexture(Material::TextureType::DIFFUSE, textureName); + } - fs::path texturePath = outPath / textureName; + const fs::path texturePath = outPath / textureName; ALICEVISION_LOG_INFO(" - Writing texture file: " << texturePath.string()); image::writeImage(texturePath.string(), @@ -1206,123 +1465,6 @@ void Texturing::saveAs(const fs::path& dir, const std::string& basename, EFileTy ALICEVISION_LOG_INFO("Save mesh to " << meshFileTypeStr << " done."); } -inline GEO::vec3 mesh_facet_interpolate_normal_at_point(const GEO::Mesh& mesh, GEO::index_t f, const GEO::vec3& p) -{ - const GEO::index_t v0 = mesh.facets.vertex(f, 0); - const GEO::index_t v1 = mesh.facets.vertex(f, 1); - const GEO::index_t v2 = mesh.facets.vertex(f, 2); - - const GEO::vec3 p0 = mesh.vertices.point(v0); - const GEO::vec3 p1 = mesh.vertices.point(v1); - const GEO::vec3 p2 = mesh.vertices.point(v2); - - const GEO::vec3 n0 = GEO::normalize(GEO::Geom::mesh_vertex_normal(mesh, v0)); - const GEO::vec3 n1 = GEO::normalize(GEO::Geom::mesh_vertex_normal(mesh, v1)); - const GEO::vec3 n2 = GEO::normalize(GEO::Geom::mesh_vertex_normal(mesh, v2)); - - GEO::vec3 barycCoords; - GEO::vec3 closestPoint; - GEO::Geom::point_triangle_squared_distance(p, p0, p1, p2, closestPoint, barycCoords.x, barycCoords.y, barycCoords.z); - - const GEO::vec3 n = barycCoords.x * n0 + barycCoords.y * n1 + barycCoords.z * n2; - - return GEO::normalize(n); -} - -inline GEO::vec3 mesh_facet_interpolate_normal_at_point(const StaticVector& ptsNormals, const Mesh& mesh, GEO::index_t f, const GEO::vec3& p) -{ - const GEO::index_t v0 = (mesh.tris)[f].v[0]; - const GEO::index_t v1 = (mesh.tris)[f].v[1]; - const GEO::index_t v2 = (mesh.tris)[f].v[2]; - - const GEO::vec3 p0((mesh.pts)[v0].x, (mesh.pts)[v0].y, (mesh.pts)[v0].z); - const GEO::vec3 p1((mesh.pts)[v1].x, (mesh.pts)[v1].y, (mesh.pts)[v1].z); - const GEO::vec3 p2((mesh.pts)[v2].x, (mesh.pts)[v2].y, (mesh.pts)[v2].z); - - const GEO::vec3 n0(ptsNormals[v0].x, ptsNormals[v0].y, ptsNormals[v0].z); - const GEO::vec3 n1(ptsNormals[v1].x, ptsNormals[v1].y, ptsNormals[v1].z); - const GEO::vec3 n2(ptsNormals[v2].x, ptsNormals[v2].y, ptsNormals[v2].z); - - GEO::vec3 barycCoords; - GEO::vec3 closestPoint; - GEO::Geom::point_triangle_squared_distance(p, p0, p1, p2, closestPoint, barycCoords.x, barycCoords.y, barycCoords.z); - - const GEO::vec3 n = barycCoords.x * n0 + barycCoords.y * n1 + barycCoords.z * n2; - - return GEO::normalize(n); -} - -template -inline Eigen::Matrix toEigen(const GEO::vecng& v) -{ - return Eigen::Matrix(v.data()); -} - -/** - * @brief Compute a transformation matrix to convert coordinates in world space coordinates into the triangle space. - * The triangle space is define by the Z-axis as the normal of the triangle, - * the X-axis aligned with the horizontal line in the texture file (using texture/UV coordinates). - * - * @param[in] mesh: input mesh - * @param[in] f: facet/triangle index - * @param[in] triPts: UV Coordinates - * @return Rotation matrix to convert from world space coordinates in the triangle space - */ -inline Eigen::Matrix3d computeTriangleTransform(const Mesh& mesh, int f, const Point2d* triPts) -{ - const Eigen::Vector3d p0 = toEigen((mesh.pts)[(mesh.tris)[f].v[0]]); - const Eigen::Vector3d p1 = toEigen((mesh.pts)[(mesh.tris)[f].v[1]]); - const Eigen::Vector3d p2 = toEigen((mesh.pts)[(mesh.tris)[f].v[2]]); - - const Eigen::Vector3d tX = (p1 - p0).normalized(); // edge0 => local triangle X-axis - const Eigen::Vector3d N = tX.cross((p2 - p0).normalized()).normalized(); // cross(edge0, edge1) => Z-axis - - // Correct triangle X-axis to be align with X-axis in the texture - const GEO::vec2 t0 = GEO::vec2(triPts[0].m); - const GEO::vec2 t1 = GEO::vec2(triPts[1].m); - const GEO::vec2 tV = GEO::normalize(t1 - t0); - const GEO::vec2 origNormal(1.0, 0.0); // X-axis in the texture - const double tAngle = GEO::Geom::angle(tV, origNormal); - Eigen::Matrix3d transform(Eigen::AngleAxisd(tAngle, N).toRotationMatrix()); - // Rotate triangle v0v1 axis around Z-axis, to get a X axis aligned with the 2d texture - Eigen::Vector3d X = (transform * tX).normalized(); - - const Eigen::Vector3d Y = N.cross(X).normalized(); // Y-axis - - Eigen::Matrix3d m; - m.col(0) = X; - m.col(1) = Y; - m.col(2) = N; - // const Eigen::Matrix3d mInv = m.inverse(); - const Eigen::Matrix3d mT = m.transpose(); - - return mT; -} - -inline void computeNormalHeight(const GEO::Mesh& mesh, - double orientation, - double t, - GEO::index_t f, - const Eigen::Matrix3d& m, - const GEO::vec3& q, - const GEO::vec3& qA, - const GEO::vec3& qB, - float& out_height, - image::RGBfColor& out_normal) -{ - GEO::vec3 intersectionPoint = t * qB + (1.0 - t) * qA; - out_height = q.distance(intersectionPoint) * orientation; - - // Use facet normal - // GEO::vec3 denseMeshNormal_f = normalize(GEO::Geom::mesh_facet_normal(mesh, f)); - // Use per pixel normal using weighted interpolation of the facet vertex normals - const GEO::vec3 denseMeshNormal = mesh_facet_interpolate_normal_at_point(mesh, f, intersectionPoint); - - Eigen::Vector3d dNormal = m * toEigen(denseMeshNormal); - dNormal.normalize(); - out_normal = image::RGBfColor(dNormal(0), dNormal(1), dNormal(2)); -} - void Texturing::_generateNormalAndHeightMaps(const mvsUtils::MultiViewParams& mp, const GEO::MeshFacetsAABB& denseMeshAABB, const GEO::Mesh& sparseMesh, diff --git a/src/aliceVision/mesh/Texturing.hpp b/src/aliceVision/mesh/Texturing.hpp index 601b8eb111..545ad0913b 100644 --- a/src/aliceVision/mesh/Texturing.hpp +++ b/src/aliceVision/mesh/Texturing.hpp @@ -196,14 +196,16 @@ struct Texturing void generateTextures(const mvsUtils::MultiViewParams& mp, const fs::path& outPath, size_t memoryAvailable, - image::EImageFileType textureFileType = image::EImageFileType::PNG); + image::EImageFileType textureFileType = image::EImageFileType::PNG, + mvsUtils::EFileType imageType = mvsUtils::EFileType::none); /// Generate texture files for the given sub-set of texture atlases void generateTexturesSubSet(const mvsUtils::MultiViewParams& mp, const std::vector& atlasIDs, mvsUtils::ImagesCache>& imageCache, const fs::path& outPath, - image::EImageFileType textureFileType = image::EImageFileType::PNG); + image::EImageFileType textureFileType = image::EImageFileType::PNG, + mvsUtils::EFileType imageType = mvsUtils::EFileType::none); void generateNormalAndHeightMaps(const mvsUtils::MultiViewParams& mp, const Mesh& denseMesh, @@ -223,7 +225,8 @@ struct Texturing const std::size_t atlasID, const fs::path& outPath, image::EImageFileType textureFileType, - const int level); + const int level, + mvsUtils::EFileType imageType = mvsUtils::EFileType::none); /// Save textured mesh as an OBJ + MTL file void saveAs(const fs::path& dir, const std::string& basename, aliceVision::mesh::EFileType meshFileType = aliceVision::mesh::EFileType::OBJ); diff --git a/src/aliceVision/mvsUtils/MultiViewParams.cpp b/src/aliceVision/mvsUtils/MultiViewParams.cpp index 2cb0ad67ad..a1e7f6540b 100644 --- a/src/aliceVision/mvsUtils/MultiViewParams.cpp +++ b/src/aliceVision/mvsUtils/MultiViewParams.cpp @@ -78,8 +78,7 @@ MultiViewParams::MultiViewParams(const sfmData::SfMData& sfmData, } else if (fileType == mvsUtils::EFileType::normalMap) { - const int scale = 0; - path = getFileNameFromViewId(*this, view.getViewId(), mvsUtils::EFileType::normalMap, scale); + path = getFileNameFromViewId(*this, view.getViewId(), mvsUtils::EFileType::normalMap); } else if (_imagesFolder != "/" && !_imagesFolder.empty() && fs::is_directory(_imagesFolder) && !fs::is_empty(_imagesFolder)) { diff --git a/src/software/pipeline/main_meshing.cpp b/src/software/pipeline/main_meshing.cpp index 19ffa18a62..8c8ffea046 100644 --- a/src/software/pipeline/main_meshing.cpp +++ b/src/software/pipeline/main_meshing.cpp @@ -306,7 +306,7 @@ int aliceVision_main(int argc, char* argv[]) } // initialization - mvsUtils::MultiViewParams mp(sfmData, "", "", depthMapsFolder, meshingFromDepthMaps); + mvsUtils::MultiViewParams mp(sfmData, "", "", depthMapsFolder, meshingFromDepthMaps ? mvsUtils::EFileType::depthMap : mvsUtils::EFileType::none); mp.userParams.put("LargeScale.universePercentile", universePercentile); mp.userParams.put("LargeScale.helperPointsGridSize", helperPointsGridSize); diff --git a/src/software/pipeline/main_texturing.cpp b/src/software/pipeline/main_texturing.cpp index 0be28738fe..6e293b57d7 100644 --- a/src/software/pipeline/main_texturing.cpp +++ b/src/software/pipeline/main_texturing.cpp @@ -50,6 +50,8 @@ int aliceVision_main(int argc, char* argv[]) std::string outputFolder; std::string imagesFolder; + std::string normalsFolder; + image::EImageColorSpace workingColorSpace = image::EImageColorSpace::SRGB; image::EImageColorSpace outputColorSpace = image::EImageColorSpace::AUTO; bool flipNormals = false; @@ -81,6 +83,9 @@ int aliceVision_main(int argc, char* argv[]) ("imagesFolder", po::value(&imagesFolder), "Use images from a specific folder instead of those specify in the SfMData file.\n" "Filename should be the image UID.") + ("normalsFolder", po::value(&normalsFolder), + "Use normal maps from a specific folder to texture the mesh.\n" + "Filename should be: UID_normalMap.") ("textureSide", po::value(&texParams.textureSide)->default_value(texParams.textureSide), "Output texture size.") ("downscale", po::value(&texParams.downscale)->default_value(texParams.downscale), @@ -98,9 +103,9 @@ int aliceVision_main(int argc, char* argv[]) ("bumpType", po::value(&bumpMappingParams.bumpType)->default_value(bumpMappingParams.bumpType), "Use HeightMap for displacement or bump mapping.") ("unwrapMethod", po::value(&unwrapMethod)->default_value(unwrapMethod), - "Method to unwrap input mesh if it does not have UV coordinates.\n" - " * Basic (> 600k faces) fast and simple. Can generate multiple atlases.\n" - " * LSCM (<= 600k faces): optimize space. Generates one atlas.\n" + "Method to unwrap input mesh if it does not have UV coordinates.\n" + " * Basic (> 600k faces) fast and simple. Can generate multiple atlases.\n" + " * LSCM (<= 600k faces): optimize space. Generates one atlas.\n" " * ABF (<= 300k faces): optimize space and stretch. Generates one atlas.'") ("useUDIM", po::value(&texParams.useUDIM)->default_value(texParams.useUDIM), "Use UDIM UV mapping.") @@ -130,9 +135,9 @@ int aliceVision_main(int argc, char* argv[]) "Option to flip face normals. It can be needed as it depends on the vertices order in triangles and the " "convention changes from one software to another.") ("visibilityRemappingMethod", po::value(&visibilityRemappingMethod)->default_value(visibilityRemappingMethod), - "Method to remap visibilities from the reconstruction to the input mesh.\n" - " * Pull: For each vertex of the input mesh, pull the visibilities from the closest vertex in the reconstruction.\n" - " * Push: For each vertex of the reconstruction, push the visibilities to the closest triangle in the input mesh.\n" + "Method to remap visibilities from the reconstruction to the input mesh.\n" + " * Pull: For each vertex of the input mesh, pull the visibilities from the closest vertex in the reconstruction.\n" + " * Push: For each vertex of the reconstruction, push the visibilities to the closest triangle in the input mesh.\n" " * PullPush: Combine results from Pull and Push results.'") ("subdivisionTargetRatio", po::value(&texParams.subdivisionTargetRatio)->default_value(texParams.subdivisionTargetRatio), "Percentage of the density of the reconstruction as the target for the subdivision " @@ -248,6 +253,14 @@ int aliceVision_main(int argc, char* argv[]) mesh.generateNormalAndHeightMaps(mp, denseMesh, outputFolder, bumpMappingParams); } + // generate normal maps textures from the fusion of normal maps per image + if (!normalsFolder.empty() && bumpMappingParams.bumpMappingFileType != image::EImageFileType::NONE) + { + ALICEVISION_LOG_INFO("Generate normal maps."); + mvsUtils::MultiViewParams mpN(sfmData, normalsFolder); + mesh.generateTextures(mpN, outputFolder, hwc.getMaxMemory(), texParams.textureFileType, mvsUtils::EFileType::normalMap); + } + // save final obj file if (!inputMeshFilepath.empty()) { From 13ee765c9068652eee4805602e2e5534d2ccb54c Mon Sep 17 00:00:00 2001 From: jmelou Date: Mon, 18 Sep 2023 17:18:21 +0200 Subject: [PATCH 12/67] [texturing] Debug check for pixels that had been visited before --- src/aliceVision/mesh/Texturing.cpp | 30 +++++++++--------------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/src/aliceVision/mesh/Texturing.cpp b/src/aliceVision/mesh/Texturing.cpp index fb55dcaa70..e58c519d6e 100644 --- a/src/aliceVision/mesh/Texturing.cpp +++ b/src/aliceVision/mesh/Texturing.cpp @@ -863,12 +863,13 @@ void Texturing::generateTexturesSubSet(const mvsUtils::MultiViewParams& mp, // If mode "normalMaps" if (imageType == mvsUtils::EFileType::normalMap) { + Eigen::Matrix visitedPixels(atlasTexture.img.rows(), atlasTexture.img.cols()); + visitedPixels.fill(false); + // Rotation and normalization of normals #pragma omp parallel for for (size_t i = 0; i < _atlases[atlasID].size(); ++i) { - // Eigen::Matrix visitedPixels(atlasTexture.img.rows(), atlasTexture.img.cols()); - // visitedPixels.fill(false); int triangleId = _atlases[atlasID][i]; // Retrieve triangle 3D and UV coordinates @@ -931,18 +932,6 @@ void Texturing::generateTexturesSubSet(const mvsUtils::MultiViewParams& mp, continue; } - // bool visited; - // #pragma omp critical - // { - // visited = visitedPixels(x,y); - // visitedPixels(x,y) = true; - // } - - // if(visited) - // { - // continue; - // } - // remap 'y' to image coordinates system (inverted Y axis) const unsigned int y_ = (texParams.textureSide - 1) - y; // 1D pixel index @@ -962,18 +951,17 @@ void Texturing::generateTexturesSubSet(const mvsUtils::MultiViewParams& mp, */ Vec3 origNormal = atlasTexture.img(xyoffset).cast(); - if (origNormal[2] > 0) + if (visitedPixels(x, y)) { continue; } - origNormal.normalize(); + visitedPixels(x, y) = true; + origNormal = worldToTriangleMatrix * origNormal; - atlasTexture.img(xyoffset) = image::RGBfColor(origNormal[0], origNormal[1], origNormal[2]); + origNormal.normalize(); - // Normal in visual representation - /*normalMap(i) = image::RGBfColor(normalMap(i).r() * 0.5 + 0.5, - normalMap(i).g() * 0.5 + 0.5, - normalMap(i).b() * 0.5 + 0.5); // B: -1:+1 => 0-255 which means 0:+1 => 128-255*/ + origNormal = origNormal * 0.5 + Vec3(0.5, 0.5, 0.5); // Normal in visual representation + atlasTexture.img(xyoffset) = image::RGBfColor(origNormal[0], origNormal[1], origNormal[2]); } } } From 55b458b1d6264f8bae19798d3b44af434ff254a0 Mon Sep 17 00:00:00 2001 From: jmelou Date: Mon, 25 Sep 2023 16:35:57 +0200 Subject: [PATCH 13/67] [lightCalibration] Debug read lp and json lightfiles --- src/aliceVision/photometricStereo/photometricDataIO.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/aliceVision/photometricStereo/photometricDataIO.cpp b/src/aliceVision/photometricStereo/photometricDataIO.cpp index 2b0221ae1e..676708d0e4 100644 --- a/src/aliceVision/photometricStereo/photometricDataIO.cpp +++ b/src/aliceVision/photometricStereo/photometricDataIO.cpp @@ -195,6 +195,9 @@ void buildLightMatFromJSON(const std::string& fileName, int cpt = 0; for (auto& light : fileTree.get_child("lights")) { + if (lineNumber == lightMat.rows()) + break; + IndexT lightIndex = light.second.get("lightId", UndefinedIndexT); if (lightIndex != UndefinedIndexT) { @@ -205,14 +208,13 @@ void buildLightMatFromJSON(const std::string& fileName, ++cpt; } intList.push_back(currentIntensities); - cpt = 0; for (auto& direction : light.second.get_child("direction")) { - lightMat(lightIndex, cpt) = direction.second.get_value(); + lightMat(lineNumber, cpt) = direction.second.get_value(); ++cpt; } - break; + ++lineNumber; } else { From 4ea4acf7457d5f2c9331752ea8c32f1b1d36e1b9 Mon Sep 17 00:00:00 2001 From: jmelou Date: Sun, 29 Oct 2023 18:29:46 +0100 Subject: [PATCH 14/67] [lightingCalibration] Check for metadata --- .../lightingEstimation/lightingCalibration.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/aliceVision/lightingEstimation/lightingCalibration.cpp b/src/aliceVision/lightingEstimation/lightingCalibration.cpp index 4c3f6f681e..00545567a0 100644 --- a/src/aliceVision/lightingEstimation/lightingCalibration.cpp +++ b/src/aliceVision/lightingEstimation/lightingCalibration.cpp @@ -54,7 +54,16 @@ void lightCalibration(const sfmData::SfMData& sfmData, for (auto& viewIt : sfmData.getViews()) { std::map currentMetadata = sfmData.getView(viewIt.first).getImage().getMetadata(); - viewMap[currentMetadata.at("Exif:DateTimeDigitized")] = sfmData.getView(viewIt.first); + + if (currentMetadata.find("Exif:DateTimeDigitized") == currentMetadata.end()) + { + std::cout << "No metadata case" << std::endl; + viewMap[sfmData.getView(viewIt.first).getImage().getImagePath()] = sfmData.getView(viewIt.first); + } + else + { + viewMap[currentMetadata.at("Exif:DateTimeDigitized")] = sfmData.getView(viewIt.first); + } } for (const auto& [currentTime, currentView] : viewMap) From f06c170127cfe7b7cfa019ee443a6d98de21af99 Mon Sep 17 00:00:00 2001 From: jmelou Date: Sun, 29 Oct 2023 18:30:24 +0100 Subject: [PATCH 15/67] [lighting calibration] Check for metadata (part. II) --- .../lightingEstimation/lightingCalibration.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/aliceVision/lightingEstimation/lightingCalibration.cpp b/src/aliceVision/lightingEstimation/lightingCalibration.cpp index 00545567a0..1e147fdc4f 100644 --- a/src/aliceVision/lightingEstimation/lightingCalibration.cpp +++ b/src/aliceVision/lightingEstimation/lightingCalibration.cpp @@ -287,7 +287,16 @@ void writeJSON(const std::string& fileName, for (auto& viewIt : sfmData.getViews()) { std::map currentMetadata = sfmData.getView(viewIt.first).getImage().getMetadata(); - viewMap[currentMetadata.at("Exif:DateTimeDigitized")] = sfmData.getView(viewIt.first); + + if (currentMetadata.find("Exif:DateTimeDigitized") == currentMetadata.end()) + { + std::cout << "No metadata case" << std::endl; + viewMap[sfmData.getView(viewIt.first).getImage().getImagePath()] = sfmData.getView(viewIt.first); + } + else + { + viewMap[currentMetadata.at("Exif:DateTimeDigitized")] = sfmData.getView(viewIt.first); + } } for (const auto& [currentTime, viewId] : viewMap) From ce5f74af9afea48cd53eeb9d77ae929cebc9bfd1 Mon Sep 17 00:00:00 2001 From: jmelou Date: Sun, 29 Oct 2023 18:32:37 +0100 Subject: [PATCH 16/67] [lighting calibration] Add debug function Write picture on wich is draw the sphere with the estimated lighting --- .../lightingCalibration.cpp | 47 +++++++++++++++++++ .../lightingCalibration.hpp | 2 + 2 files changed, 49 insertions(+) diff --git a/src/aliceVision/lightingEstimation/lightingCalibration.cpp b/src/aliceVision/lightingEstimation/lightingCalibration.cpp index 1e147fdc4f..ffe2f40197 100644 --- a/src/aliceVision/lightingEstimation/lightingCalibration.cpp +++ b/src/aliceVision/lightingEstimation/lightingCalibration.cpp @@ -356,5 +356,52 @@ void writeJSON(const std::string& fileName, bpt::write_json(fileName, fileTree); } +void sphereFromLighting(const Eigen::VectorXf& lightVector, const float intensity, const std::string outputFileName, const int outputSize) +{ + float radius = (outputSize * 0.9) / 2; + image::Image pixelsValues(outputSize, outputSize); + + for (size_t j = 0; j < outputSize; ++j) + { + for (size_t i = 0; i < outputSize; ++i) + { + float center_xy = outputSize / 2; + Eigen::VectorXf normalSphere(lightVector.size()); + float distanceToCenter = sqrt((i - center_xy) * (i - center_xy) + (j - center_xy) * (j - center_xy)); + pixelsValues(i, j) = 0; + + if (distanceToCenter < radius) + { + normalSphere(0) = (float(j) - center_xy) / radius; + normalSphere(1) = (float(i) - center_xy) / radius; + normalSphere(2) = -sqrt(1 - normalSphere(0) * normalSphere(0) - normalSphere(1) * normalSphere(1)); + if (lightVector.size() > 3) + { + normalSphere(3) = 1; + } + if (lightVector.size() > 4) + { + normalSphere(4) = normalSphere(0) * normalSphere(1); + normalSphere(5) = normalSphere(0) * normalSphere(2); + normalSphere(6) = normalSphere(1) * normalSphere(2); + normalSphere(7) = normalSphere(0) * normalSphere(0) - normalSphere(1) * normalSphere(1); + normalSphere(8) = 3 * normalSphere(2) * normalSphere(2) - 1; + } + + for (size_t k = 0; k < lightVector.size(); ++k) + { + pixelsValues(i, j) += normalSphere(k) * lightVector(k); + } + pixelsValues(i, j) *= intensity; + } + } + } + + image::writeImage( + outputFileName, + pixelsValues, + image::ImageWriteOptions().toColorSpace(image::EImageColorSpace::NO_CONVERSION).storageDataType(image::EStorageDataType::Float)); +} + } // namespace lightingEstimation } // namespace aliceVision diff --git a/src/aliceVision/lightingEstimation/lightingCalibration.hpp b/src/aliceVision/lightingEstimation/lightingCalibration.hpp index 6dfd0a0044..1eeff43910 100644 --- a/src/aliceVision/lightingEstimation/lightingCalibration.hpp +++ b/src/aliceVision/lightingEstimation/lightingCalibration.hpp @@ -95,5 +95,7 @@ void writeJSON(const std::string& fileName, const std::vector& intList, const bool saveAsModel); +void sphereFromLighting(const Eigen::VectorXf& lightVector, const float intensity, const std::string outputFileName, const int outputSize); + } // namespace lightingEstimation } // namespace aliceVision From a6800eb134ade197b590f87f83d033ac3d819313 Mon Sep 17 00:00:00 2001 From: jmelou Date: Sun, 29 Oct 2023 18:33:28 +0100 Subject: [PATCH 17/67] [ps] Typo --- src/software/pipeline/main_photometricStereo.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/software/pipeline/main_photometricStereo.cpp b/src/software/pipeline/main_photometricStereo.cpp index 0fd4da18b4..674671912b 100644 --- a/src/software/pipeline/main_photometricStereo.cpp +++ b/src/software/pipeline/main_photometricStereo.cpp @@ -118,7 +118,6 @@ int aliceVision_main(int argc, char** argv) ALICEVISION_LOG_ERROR("The input file '" + inputPath + "' cannot be read."); return EXIT_FAILURE; } - photometricStereo::photometricStereo(sfmData, pathToLightData, maskPath, outputPath, PSParameters, normalsIm, albedoIm); } From 664cc6888bff23d91bde5c0432c1865154d40edd Mon Sep 17 00:00:00 2001 From: jmelou Date: Sun, 29 Oct 2023 18:34:51 +0100 Subject: [PATCH 18/67] [ps] Add log, indent --- src/aliceVision/photometricStereo/photometricStereo.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/aliceVision/photometricStereo/photometricStereo.cpp b/src/aliceVision/photometricStereo/photometricStereo.cpp index c3b88f6aa9..bf1aea5843 100644 --- a/src/aliceVision/photometricStereo/photometricStereo.cpp +++ b/src/aliceVision/photometricStereo/photometricStereo.cpp @@ -147,6 +147,7 @@ void photometricStereo(const sfmData::SfMData& sfmData, } else if (PSParameters.removeAmbiant) { + ALICEVISION_LOG_INFO("Remove ambiant light - " << imagePath.string()); pathToAmbiant = imagePath.string(); } } From 5bd31cace4d55c5a407f29befbca0bdf37bde87a Mon Sep 17 00:00:00 2001 From: jmelou Date: Sun, 29 Oct 2023 18:36:29 +0100 Subject: [PATCH 19/67] [ps] Add png output --- .../photometricStereo/photometricDataIO.cpp | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/aliceVision/photometricStereo/photometricDataIO.cpp b/src/aliceVision/photometricStereo/photometricDataIO.cpp index 676708d0e4..a6e64b6354 100644 --- a/src/aliceVision/photometricStereo/photometricDataIO.cpp +++ b/src/aliceVision/photometricStereo/photometricDataIO.cpp @@ -489,6 +489,14 @@ void writePSResults(const std::string& outputPath, const image::Image normalsImPNG(normals.cols(), normals.rows()); + convertNormalMap2png(normals, normalsImPNG); + image::writeImage( + outputPath + "/normals.png", + normalsImPNG, + image::ImageWriteOptions().toColorSpace(image::EImageColorSpace::NO_CONVERSION).storageDataType(image::EStorageDataType::Float)); + image::writeImage( outputPath + "/albedo.exr", albedo, @@ -504,6 +512,14 @@ void writePSResults(const std::string& outputPath, outputPath + "/" + std::to_string(poseId) + "_normals.exr", normals, image::ImageWriteOptions().toColorSpace(image::EImageColorSpace::NO_CONVERSION).storageDataType(image::EStorageDataType::Float)); + + image::Image normalsImPNG(normals.cols(), normals.rows()); + convertNormalMap2png(normals, normalsImPNG); + image::writeImage( + outputPath + "/" + std::to_string(poseId) + "_normals.png", + normalsImPNG, + image::ImageWriteOptions().toColorSpace(image::EImageColorSpace::NO_CONVERSION).storageDataType(image::EStorageDataType::Float)); + image::writeImage( outputPath + "/" + std::to_string(poseId) + "_albedo.exr", albedo, From f6417855222dd89acbb29dcadc422241d91ccdb6 Mon Sep 17 00:00:00 2001 From: jmelou Date: Mon, 30 Oct 2023 09:52:39 +0100 Subject: [PATCH 20/67] [ps] Resolve issue in mask values --- .../photometricStereo/photometricStereo.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/aliceVision/photometricStereo/photometricStereo.cpp b/src/aliceVision/photometricStereo/photometricStereo.cpp index bf1aea5843..62fef2fa9c 100644 --- a/src/aliceVision/photometricStereo/photometricStereo.cpp +++ b/src/aliceVision/photometricStereo/photometricStereo.cpp @@ -466,11 +466,11 @@ void photometricStereo(const std::vector& imageList, } } - for (size_t i = 0; i < maskSize; ++i) + for (size_t i = 0; i < currentMaskSize; ++i) { if (hasMask) { - currentIdx = indices.at(i); // Index in picture + currentIdx = currentMaskIndices.at(i); // Index in picture } else { @@ -483,17 +483,17 @@ void photometricStereo(const std::vector& imageList, for (size_t ch = 0; ch < 3; ++ch) { // Create I matrix for current pixel - Eigen::MatrixXf pixelValues_channel(imageList.size(), maskSize); + Eigen::MatrixXf pixelValues_channel(imageList.size(), currentMaskSize); for (size_t i = 0; i < imageList.size(); ++i) { - pixelValues_channel.block(i, 0, 1, maskSize) = imMat.block(ch + 3 * i, 0, 1, maskSize); + pixelValues_channel.block(i, 0, 1, currentMaskSize) = imMat.block(ch + 3 * i, 0, 1, currentMaskSize); } - for (size_t i = 0; i < maskSize; ++i) + for (size_t i = 0; i < currentMaskSize; ++i) { if (hasMask) { - currentIdx = indices.at(i); // Index in picture + currentIdx = currentMaskIndices.at(i); // Index in picture } else { From af00331ab1895cb6e458ee62792b3e4c41cda49b Mon Sep 17 00:00:00 2001 From: jmelou Date: Mon, 30 Oct 2023 09:53:11 +0100 Subject: [PATCH 21/67] [ps] resolve issue in mask values (part. 2) --- src/aliceVision/photometricStereo/photometricStereo.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/aliceVision/photometricStereo/photometricStereo.cpp b/src/aliceVision/photometricStereo/photometricStereo.cpp index 62fef2fa9c..03c67a7ffd 100644 --- a/src/aliceVision/photometricStereo/photometricStereo.cpp +++ b/src/aliceVision/photometricStereo/photometricStereo.cpp @@ -528,10 +528,10 @@ void photometricStereo(const std::vector& imageList, for (size_t ch = 0; ch < 3; ++ch) { // Create I matrix for current pixel - Eigen::MatrixXf pixelValues_channel(imageList.size(), maskSize); + Eigen::MatrixXf pixelValues_channel(imageList.size(), currentMaskSize); for (size_t i = 0; i < imageList.size(); ++i) { - pixelValues_channel.block(i, 0, 1, maskSize) = imMat.block(ch + 3 * i, 0, 1, maskSize); + pixelValues_channel.block(i, 0, 1, currentMaskSize) = imMat.block(ch + 3 * i, 0, 1, currentMaskSize); } M_channel = lightMat.bdcSvd(Eigen::ComputeThinU | Eigen::ComputeThinV).solve(pixelValues_channel); From 1ecf28263a4eb1941f6cdc1d97f027f4432f7f3d Mon Sep 17 00:00:00 2001 From: jmelou Date: Mon, 30 Oct 2023 15:34:38 +0100 Subject: [PATCH 22/67] [lighting calibration] Add spherical harmonics estimation - New lighting estimation based on spherical harmonics model. - Light's direction and intensity are now evaluated in the same function - New field in the Light file explaining the method used - SH in PS available soon --- .../lightingCalibration.cpp | 105 +++++++++++++++--- .../lightingCalibration.hpp | 7 +- .../photometricStereo/photometricDataIO.cpp | 28 ++++- .../photometricStereo/photometricStereo.cpp | 12 +- 4 files changed, 126 insertions(+), 26 deletions(-) diff --git a/src/aliceVision/lightingEstimation/lightingCalibration.cpp b/src/aliceVision/lightingEstimation/lightingCalibration.cpp index ffe2f40197..e8fa3c3975 100644 --- a/src/aliceVision/lightingEstimation/lightingCalibration.cpp +++ b/src/aliceVision/lightingEstimation/lightingCalibration.cpp @@ -81,7 +81,6 @@ void lightCalibration(const sfmData::SfMData& sfmData, imageList.push_back(imagePath.string()); std::array currentSphereParams; - for (auto& currentSphere : fileTree.get_child(sphereName)) { currentSphereParams[0] = currentSphere.second.get_child("").get("x", 0.0); @@ -101,7 +100,11 @@ void lightCalibration(const sfmData::SfMData& sfmData, } } - Eigen::MatrixXf lightMat(imageList.size(), 3); + int lightSize = 3; + if (!method.compare("HS")) + lightSize = 9; + + Eigen::MatrixXf lightMat(imageList.size(), lightSize); std::vector intList; for (size_t i = 0; i < imageList.size(); ++i) @@ -110,21 +113,24 @@ void lightCalibration(const sfmData::SfMData& sfmData, std::array sphereParam = allSpheresParams.at(i); float focal = focals.at(i); - Eigen::Vector3f lightingDirection; - lightCalibrationOneImage(picturePath, sphereParam, focal, method, lightingDirection); + Eigen::VectorXf lightingDirection = Eigen::VectorXf::Zero(lightSize); + float intensity; + lightCalibrationOneImage(picturePath, sphereParam, focal, method, lightingDirection, intensity); + lightMat.row(i) = lightingDirection; - intList.push_back(lightingDirection.norm()); + intList.push_back(intensity); } // Write in JSON file - writeJSON(outputPath, sfmData, imageList, lightMat, intList, saveAsModel); + writeJSON(outputPath, sfmData, imageList, lightMat, intList, saveAsModel, method); } void lightCalibrationOneImage(const std::string& picturePath, const std::array& sphereParam, const float focal, const std::string& method, - Eigen::Vector3f& lightingDirection) + Eigen::VectorXf& lightingDirection, + float& intensity) { // Read picture : image::Image imageFloat; @@ -151,8 +157,10 @@ void lightCalibrationOneImage(const std::string& picturePath, // Evaluate lighting direction : lightingDirection = 2 * normalBrightestPoint.dot(observationRayPersp) * normalBrightestPoint - observationRayPersp; lightingDirection = lightingDirection / lightingDirection.norm(); + + intensity = 1.0; } - // If method = HS : + // If method = whiteSphere : else if (!method.compare("whiteSphere")) { // Evaluate light direction and intensity by pseudo-inverse @@ -174,8 +182,8 @@ void lightCalibrationOneImage(const std::string& picturePath, { for (int i = 0; i < patch.rows(); ++i) { - float distanceToCenter = (i - radius) * (i - radius) + (j - radius) * (j - radius); - if ((distanceToCenter < (radius * radius - 0.05 * radius)) && (patch(i, j) > 0.3) && (patch(i, j) < 0.8)) + const float distanceToCenter = std::sqrt((i - radius) * (i - radius) + (j - radius) * (j - radius)); + if ((distanceToCenter < 0.95 * radius) && (patch(i, j) > 0.1) && (patch(i, j) < 0.98)) { // imSphere = normalSphere.s imSphere(currentIndex) = patch(i, j); @@ -189,12 +197,74 @@ void lightCalibrationOneImage(const std::string& picturePath, } } } + Eigen::MatrixXf normalSphereMasked(currentIndex, 3); normalSphereMasked = normalSphere.block(0, 0, currentIndex, 3); Eigen::VectorXf imSphereMasked(currentIndex); imSphereMasked = imSphere.head(currentIndex); lightingDirection = normalSphere.colPivHouseholderQr().solve(imSphere); + + intensity = lightingDirection.norm(); + lightingDirection = lightingDirection / intensity; + } + + // If method = HS : + else if (!method.compare("HS")) + { + size_t lightSize = lightingDirection.size(); + + // Evaluate light direction and intensity by pseudo-inverse + int minISphere = floor(sphereParam[1] - sphereParam[2] + imageFloat.rows() / 2); + int minJSphere = floor(sphereParam[0] - sphereParam[2] + imageFloat.cols() / 2); + + float radius = sphereParam[2]; + + image::Image patch; + patch = imageFloat.block(minISphere, minJSphere, 2 * radius, 2 * radius); + + int nbPixelsPatch = 4 * radius * radius; + Eigen::VectorXf imSphere(nbPixelsPatch); + Eigen::MatrixXf normalSphere(nbPixelsPatch, lightSize); + + int currentIndex = 0; + + for (size_t j = 0; j < patch.cols(); ++j) + { + for (size_t i = 0; i < patch.rows(); ++i) + { + float distanceToCenter = sqrt((i - radius) * (i - radius) + (j - radius) * (j - radius)); + if (distanceToCenter < 0.95 * radius && (patch(i, j) > 0.1) && (patch(i, j) < 0.98)) + { + imSphere(currentIndex) = patch(i, j); + + normalSphere(currentIndex, 0) = (float(j) - radius) / radius; + normalSphere(currentIndex, 1) = (float(i) - radius) / radius; + normalSphere(currentIndex, 2) = -sqrt(1 - normalSphere(currentIndex, 0) * normalSphere(currentIndex, 0) - + normalSphere(currentIndex, 1) * normalSphere(currentIndex, 1)); + normalSphere(currentIndex, 3) = 1; + if (lightSize > 4) + { + normalSphere(currentIndex, 4) = normalSphere(currentIndex, 0) * normalSphere(currentIndex, 1); + normalSphere(currentIndex, 5) = normalSphere(currentIndex, 0) * normalSphere(currentIndex, 2); + normalSphere(currentIndex, 6) = normalSphere(currentIndex, 1) * normalSphere(currentIndex, 2); + normalSphere(currentIndex, 7) = normalSphere(currentIndex, 0) * normalSphere(currentIndex, 0) - + normalSphere(currentIndex, 1) * normalSphere(currentIndex, 1); + normalSphere(currentIndex, 8) = 3 * normalSphere(currentIndex, 2) * normalSphere(currentIndex, 2) - 1; + } + ++currentIndex; + } + } + } + + Eigen::MatrixXf normalSphereMasked(currentIndex, lightSize); + normalSphereMasked = normalSphere.block(0, 0, currentIndex, lightSize); + + Eigen::VectorXf imSphereMasked(currentIndex); + imSphereMasked = imSphere.head(currentIndex); + + lightingDirection = normalSphereMasked.colPivHouseholderQr().solve(imSphereMasked); + intensity = lightingDirection.head(3).norm(); } } @@ -277,7 +347,8 @@ void writeJSON(const std::string& fileName, const std::vector& imageList, const Eigen::MatrixXf& lightMat, const std::vector& intList, - const bool saveAsModel) + const bool saveAsModel, + const std::string method) { bpt::ptree lightsTree; bpt::ptree fileTree; @@ -290,7 +361,7 @@ void writeJSON(const std::string& fileName, if (currentMetadata.find("Exif:DateTimeDigitized") == currentMetadata.end()) { - std::cout << "No metadata case" << std::endl; + ALICEVISION_LOG_INFO("No metadata case (Exif:DateTimeDigitized is missing)"); viewMap[sfmData.getView(viewIt.first).getImage().getImagePath()] = sfmData.getView(viewIt.first); } else @@ -313,18 +384,20 @@ void writeJSON(const std::string& fileName, if (saveAsModel) { lightTree.put("lightId", imgCpt); - lightTree.put("type", "directional"); } else { - // const IndexT index = viewId.getViewId lightTree.put("viewId", viewId.getViewId()); - lightTree.put("type", "directional"); } + if (!method.compare("HS")) + lightTree.put("type", "HS"); + else + lightTree.put("type", "directional"); // Light direction bpt::ptree directionNode; - for (int i = 0; i < 3; ++i) + int lightMatSize = lightMat.cols(); + for (int i = 0; i < lightMatSize; ++i) { bpt::ptree cell; cell.put_value(lightMat(imgCpt, i)); diff --git a/src/aliceVision/lightingEstimation/lightingCalibration.hpp b/src/aliceVision/lightingEstimation/lightingCalibration.hpp index 1eeff43910..3fdb7369da 100644 --- a/src/aliceVision/lightingEstimation/lightingCalibration.hpp +++ b/src/aliceVision/lightingEstimation/lightingCalibration.hpp @@ -40,7 +40,8 @@ void lightCalibrationOneImage(const std::string& picturePath, const std::array& sphereParam, const float focal, const std::string& method, - Eigen::Vector3f& lightingDirection); + Eigen::VectorXf& lightingDirection, + float& intensity); /** * @brief Compute the brightest point on a sphere @@ -87,13 +88,15 @@ void cutImage(const image::Image& imageFloat, * @param[in] lightMat A matrix containing the directions of the light sources * @param[in] intList A vector of arrays containing the intensity of the light sources * @param[in] saveAsModel True to save the light IDs instead of the view IDs, false otherwise + * @param[in] method Name of the method for lighting estimation: whiteSphere, brightestPoint, SH */ void writeJSON(const std::string& fileName, const sfmData::SfMData& sfmData, const std::vector& imageList, const Eigen::MatrixXf& lightMat, const std::vector& intList, - const bool saveAsModel); + const bool saveAsModel, + const std::string method); void sphereFromLighting(const Eigen::VectorXf& lightVector, const float intensity, const std::string outputFileName, const int outputSize); diff --git a/src/aliceVision/photometricStereo/photometricDataIO.cpp b/src/aliceVision/photometricStereo/photometricDataIO.cpp index a6e64b6354..cee44edc96 100644 --- a/src/aliceVision/photometricStereo/photometricDataIO.cpp +++ b/src/aliceVision/photometricStereo/photometricDataIO.cpp @@ -211,7 +211,19 @@ void buildLightMatFromJSON(const std::string& fileName, cpt = 0; for (auto& direction : light.second.get_child("direction")) { - lightMat(lineNumber, cpt) = direction.second.get_value(); + if (cpt < 3) + { + lightMat(lineNumber, cpt) = direction.second.get_value(); + } + else if (cpt == 4) + { + // no support for SH lighting : + for (int pictureIndex = 0; pictureIndex < lightMat.rows(); ++pictureIndex) + { + lightMat.row(pictureIndex) = lightMat.row(pictureIndex) / lightMat.row(pictureIndex).norm(); + } + ALICEVISION_LOG_INFO("SH will soon be available for use in PS. For now, lighting is reduced to directional"); + } ++cpt; } ++lineNumber; @@ -233,7 +245,19 @@ void buildLightMatFromJSON(const std::string& fileName, for (auto& direction : light.second.get_child("direction")) { - lightMat(lineNumber, cpt) = direction.second.get_value(); + if (cpt < 3) + { + lightMat(lineNumber, cpt) = direction.second.get_value(); + } + else if (cpt == 4) + { + // no support for SH lighting : + for (int pictureIndex = 0; pictureIndex < lightMat.rows(); ++pictureIndex) + { + lightMat.row(pictureIndex) = lightMat.row(pictureIndex) / lightMat.row(pictureIndex).norm(); + } + ALICEVISION_LOG_INFO("SH will soon be available for use in PS. For now, lighting is reduced to directional"); + } ++cpt; } ++lineNumber; diff --git a/src/aliceVision/photometricStereo/photometricStereo.cpp b/src/aliceVision/photometricStereo/photometricStereo.cpp index 03c67a7ffd..77c8adcc77 100644 --- a/src/aliceVision/photometricStereo/photometricStereo.cpp +++ b/src/aliceVision/photometricStereo/photometricStereo.cpp @@ -35,9 +35,9 @@ void photometricStereo(const std::string& inputPath, image::Image& albedo) { size_t dim = 3; - if (PSParameters.SHOrder == 2) + if (PSParameters.SHOrder != 0) { - dim = 9; + ALICEVISION_LOG_INFO("SH will soon be available for use in PS. For now, lighting is reduced to directional"); } std::vector imageList; @@ -92,9 +92,9 @@ void photometricStereo(const sfmData::SfMData& sfmData, bool skipAll = true; bool groupedImages = false; size_t dim = 3; - if (PSParameters.SHOrder == 2) + if (PSParameters.SHOrder != 0) { - dim = 9; + ALICEVISION_LOG_INFO("SH will soon be available for use in PS. For now, lighting is reduced to directional"); } std::string pathToAmbiant = ""; @@ -358,7 +358,7 @@ void photometricStereo(const std::vector& imageList, numberOfPixels = auxMaskSize * imageList.size() * 3; } - Eigen::MatrixXf normalsVect = Eigen::MatrixXf::Zero(lightMat.cols(), pictRows * pictCols); + Eigen::MatrixXf normalsVect = Eigen::MatrixXf::Zero(3, pictRows * pictCols); Eigen::MatrixXf albedoVect = Eigen::MatrixXf::Zero(3, pictRows * pictCols); int remainingPixels = maskSize; @@ -422,7 +422,7 @@ void photometricStereo(const std::vector& imageList, currentPicture.block(2, 0, 1, currentMaskSize) * 0.0722; } - Eigen::MatrixXf M_channel(3, currentMaskSize); + Eigen::MatrixXf M_channel(lightMat.cols(), currentMaskSize); int currentIdx; if (PSParameters.isRobust) From 3fe41de42c78494d99ae839872cd5fff95e375a7 Mon Sep 17 00:00:00 2001 From: jmelou Date: Fri, 3 Nov 2023 20:18:17 +0100 Subject: [PATCH 23/67] [ps] Debug tiling --- .../photometricStereo/photometricStereo.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/aliceVision/photometricStereo/photometricStereo.cpp b/src/aliceVision/photometricStereo/photometricStereo.cpp index 77c8adcc77..3ab1905e6a 100644 --- a/src/aliceVision/photometricStereo/photometricStereo.cpp +++ b/src/aliceVision/photometricStereo/photometricStereo.cpp @@ -328,6 +328,8 @@ void photometricStereo(const std::vector& imageList, for (int i = 0; i < maskSize; ++i) indices.push_back(i); + + hasMask = true; } // Read ambiant @@ -348,14 +350,15 @@ void photometricStereo(const std::vector& imageList, // Tiling int auxMaskSize = maskSize; - int numberOfPixels = auxMaskSize * imageList.size() * 3; + int numberOfPixels = auxMaskSize * 3; + int numberOfMasks = 1; - while (numberOfPixels > sizeMax) + while (numberOfPixels > sizeMax / imageList.size()) { numberOfMasks = numberOfMasks * 2; - auxMaskSize = floor(auxMaskSize / 4); - numberOfPixels = auxMaskSize * imageList.size() * 3; + auxMaskSize = floor(auxMaskSize / 2); + numberOfPixels = 3 * auxMaskSize; } Eigen::MatrixXf normalsVect = Eigen::MatrixXf::Zero(3, pictRows * pictCols); From a82f25cbc8fb40e208b850944948bf81635f7fbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Candice=20Bent=C3=A9jac?= Date: Tue, 14 Nov 2023 10:11:55 +0100 Subject: [PATCH 24/67] [mesh] Texturing: Use integral type for OMP parallel for --- src/aliceVision/mesh/Texturing.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aliceVision/mesh/Texturing.cpp b/src/aliceVision/mesh/Texturing.cpp index e58c519d6e..1e8bf40cd2 100644 --- a/src/aliceVision/mesh/Texturing.cpp +++ b/src/aliceVision/mesh/Texturing.cpp @@ -868,7 +868,7 @@ void Texturing::generateTexturesSubSet(const mvsUtils::MultiViewParams& mp, // Rotation and normalization of normals #pragma omp parallel for - for (size_t i = 0; i < _atlases[atlasID].size(); ++i) + for (int i = 0; i < static_cast(_atlases[atlasID].size()); ++i) { int triangleId = _atlases[atlasID][i]; From af331e443ffc87f811a372d607f05bb06597f44e Mon Sep 17 00:00:00 2001 From: jmelou Date: Mon, 20 Nov 2023 21:35:26 +0100 Subject: [PATCH 25/67] [ps] Spherical Harmonics : Change "HS" in "SH --- .../lightingEstimation/lightingCalibration.cpp | 10 +++++----- .../photometricStereo/photometricDataIO.cpp | 2 +- .../photometricStereo/photometricDataIO.hpp | 2 +- .../photometricStereo/photometricStereo.cpp | 10 +++++----- .../photometricStereo/photometricStereo.hpp | 4 ++-- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/aliceVision/lightingEstimation/lightingCalibration.cpp b/src/aliceVision/lightingEstimation/lightingCalibration.cpp index e8fa3c3975..bcb59c2b27 100644 --- a/src/aliceVision/lightingEstimation/lightingCalibration.cpp +++ b/src/aliceVision/lightingEstimation/lightingCalibration.cpp @@ -101,7 +101,7 @@ void lightCalibration(const sfmData::SfMData& sfmData, } int lightSize = 3; - if (!method.compare("HS")) + if (!method.compare("SH")) lightSize = 9; Eigen::MatrixXf lightMat(imageList.size(), lightSize); @@ -209,8 +209,8 @@ void lightCalibrationOneImage(const std::string& picturePath, lightingDirection = lightingDirection / intensity; } - // If method = HS : - else if (!method.compare("HS")) + // If method = SH : + else if (!method.compare("SH")) { size_t lightSize = lightingDirection.size(); @@ -389,8 +389,8 @@ void writeJSON(const std::string& fileName, { lightTree.put("viewId", viewId.getViewId()); } - if (!method.compare("HS")) - lightTree.put("type", "HS"); + if (!method.compare("SH")) + lightTree.put("type", "SH"); else lightTree.put("type", "directional"); diff --git a/src/aliceVision/photometricStereo/photometricDataIO.cpp b/src/aliceVision/photometricStereo/photometricDataIO.cpp index cee44edc96..3df7bfb9f3 100644 --- a/src/aliceVision/photometricStereo/photometricDataIO.cpp +++ b/src/aliceVision/photometricStereo/photometricDataIO.cpp @@ -94,7 +94,7 @@ void loadLightDirections(const std::string& dirFileName, const Eigen::MatrixXf& } } -void loadLightHS(const std::string& dirFileName, Eigen::MatrixXf& lightMat) +void loadLightSH(const std::string& dirFileName, Eigen::MatrixXf& lightMat) { std::stringstream stream; std::string line; diff --git a/src/aliceVision/photometricStereo/photometricDataIO.hpp b/src/aliceVision/photometricStereo/photometricDataIO.hpp index 02d2bdc1cf..7bb4908ed4 100644 --- a/src/aliceVision/photometricStereo/photometricDataIO.hpp +++ b/src/aliceVision/photometricStereo/photometricDataIO.hpp @@ -40,7 +40,7 @@ void loadLightDirections(const std::string& dirFileName, const Eigen::MatrixXf& * @param[in] dirFileName Path to the direction file * @param[out] lightMat Matrix of directions of light */ -void loadLightHS(const std::string& dirFileName, Eigen::MatrixXf& lightMat); +void loadLightSH(const std::string& dirFileName, Eigen::MatrixXf& lightMat); /** * @brief Load light data from a JSON file to an Eigen matrix (with a list of images) diff --git a/src/aliceVision/photometricStereo/photometricStereo.cpp b/src/aliceVision/photometricStereo/photometricStereo.cpp index 3ab1905e6a..ce1510a2bd 100644 --- a/src/aliceVision/photometricStereo/photometricStereo.cpp +++ b/src/aliceVision/photometricStereo/photometricStereo.cpp @@ -566,7 +566,7 @@ void photometricStereo(const std::vector& imageList, albedo = albedoIm; } -void loadPSData(const std::string& folderPath, const size_t HS_order, std::vector>& intList, Eigen::MatrixXf& lightMat) +void loadPSData(const std::string& folderPath, const size_t SH_order, std::vector>& intList, Eigen::MatrixXf& lightMat) { std::string intFileName; std::string pathToCM; @@ -585,15 +585,15 @@ void loadPSData(const std::string& folderPath, const size_t HS_order, std::vecto } // Light directions - if (HS_order == 0) + if (SH_order == 0) { dirFileName = folderPath + "/light_directions.txt"; loadLightDirections(dirFileName, convertionMatrix, lightMat); } - else if (HS_order == 2) + else if (SH_order == 2) { - dirFileName = folderPath + "/light_directions_HS.txt"; - loadLightHS(dirFileName, lightMat); + dirFileName = folderPath + "/light_directions_SH.txt"; + loadLightSH(dirFileName, lightMat); } } diff --git a/src/aliceVision/photometricStereo/photometricStereo.hpp b/src/aliceVision/photometricStereo/photometricStereo.hpp index 4348addeb3..cd5a1eda86 100644 --- a/src/aliceVision/photometricStereo/photometricStereo.hpp +++ b/src/aliceVision/photometricStereo/photometricStereo.hpp @@ -82,11 +82,11 @@ void photometricStereo(const std::vector& imageList, /** * @brief Load data used in the PS algorithm * @param[in] folderPath Path to the folder that contains data - * @param[in] HS_order Order of the spherical harmonics + * @param[in] SH_order Order of the spherical harmonics * @param[out] intList Intensities of lights * @param[out] lightMat Directions of lights */ -void loadPSData(const std::string& folderPath, const size_t HS_order, std::vector>& intList, Eigen::MatrixXf& lightMat); +void loadPSData(const std::string& folderPath, const size_t SH_order, std::vector>& intList, Eigen::MatrixXf& lightMat); /** * @brief Get the name of the pictures in a given folder From 674e3a9962ec4b19fd47c12f52755b02a4636457 Mon Sep 17 00:00:00 2001 From: jmelou Date: Thu, 23 Nov 2023 15:49:24 +0100 Subject: [PATCH 26/67] [ps] Second version for SH calibration --- .../lightingCalibration.cpp | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/aliceVision/lightingEstimation/lightingCalibration.cpp b/src/aliceVision/lightingEstimation/lightingCalibration.cpp index bcb59c2b27..262f644ce3 100644 --- a/src/aliceVision/lightingEstimation/lightingCalibration.cpp +++ b/src/aliceVision/lightingEstimation/lightingCalibration.cpp @@ -263,8 +263,27 @@ void lightCalibrationOneImage(const std::string& picturePath, Eigen::VectorXf imSphereMasked(currentIndex); imSphereMasked = imSphere.head(currentIndex); - lightingDirection = normalSphereMasked.colPivHouseholderQr().solve(imSphereMasked); - intensity = lightingDirection.head(3).norm(); + // 1) Directionnal part estimation : + Eigen::MatrixXf normalOrdre1(currentIndex, 3); + normalOrdre1 = normalSphereMasked.leftCols(3); + Eigen::Vector3f directionnalPart = normalOrdre1.colPivHouseholderQr().solve(imSphereMasked); + intensity = directionnalPart.norm(); + directionnalPart = directionnalPart / intensity; + + // 2) Other order estimation : + Eigen::VectorXf imSphereModif(currentIndex); + imSphereModif = imSphereMasked; + for (size_t i = 0; i < currentIndex; ++i) + { + for (size_t k = 0; k < 3; ++k) + { + imSphereModif(i) -= normalSphereMasked(i, k) * directionnalPart(k); + } + } + Eigen::VectorXf secondOrder(6); + secondOrder = normalSphereMasked.rightCols(6).colPivHouseholderQr().solve(imSphereModif); + + lightingDirection << directionnalPart, secondOrder; } } From 20e91f6f64d3970cdbf7cabf2b74e5d39a6779ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Candice=20Bent=C3=A9jac?= Date: Fri, 24 Nov 2023 17:52:37 +0100 Subject: [PATCH 27/67] [software] Texturing: Fix formatting issue --- src/software/pipeline/main_texturing.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/software/pipeline/main_texturing.cpp b/src/software/pipeline/main_texturing.cpp index 6e293b57d7..9ac82a2673 100644 --- a/src/software/pipeline/main_texturing.cpp +++ b/src/software/pipeline/main_texturing.cpp @@ -103,9 +103,9 @@ int aliceVision_main(int argc, char* argv[]) ("bumpType", po::value(&bumpMappingParams.bumpType)->default_value(bumpMappingParams.bumpType), "Use HeightMap for displacement or bump mapping.") ("unwrapMethod", po::value(&unwrapMethod)->default_value(unwrapMethod), - "Method to unwrap input mesh if it does not have UV coordinates.\n" - " * Basic (> 600k faces) fast and simple. Can generate multiple atlases.\n" - " * LSCM (<= 600k faces): optimize space. Generates one atlas.\n" + "Method to unwrap input mesh if it does not have UV coordinates.\n" + " * Basic (> 600k faces) fast and simple. Can generate multiple atlases.\n" + " * LSCM (<= 600k faces): optimize space. Generates one atlas.\n" " * ABF (<= 300k faces): optimize space and stretch. Generates one atlas.'") ("useUDIM", po::value(&texParams.useUDIM)->default_value(texParams.useUDIM), "Use UDIM UV mapping.") @@ -135,9 +135,9 @@ int aliceVision_main(int argc, char* argv[]) "Option to flip face normals. It can be needed as it depends on the vertices order in triangles and the " "convention changes from one software to another.") ("visibilityRemappingMethod", po::value(&visibilityRemappingMethod)->default_value(visibilityRemappingMethod), - "Method to remap visibilities from the reconstruction to the input mesh.\n" - " * Pull: For each vertex of the input mesh, pull the visibilities from the closest vertex in the reconstruction.\n" - " * Push: For each vertex of the reconstruction, push the visibilities to the closest triangle in the input mesh.\n" + "Method to remap visibilities from the reconstruction to the input mesh.\n" + " * Pull: For each vertex of the input mesh, pull the visibilities from the closest vertex in the reconstruction.\n" + " * Push: For each vertex of the reconstruction, push the visibilities to the closest triangle in the input mesh.\n" " * PullPush: Combine results from Pull and Push results.'") ("subdivisionTargetRatio", po::value(&texParams.subdivisionTargetRatio)->default_value(texParams.subdivisionTargetRatio), "Percentage of the density of the reconstruction as the target for the subdivision " From 93e5162c82eb37cf09e89b1b9d228755a4a13783 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Candice=20Bent=C3=A9jac?= Date: Fri, 24 Nov 2023 17:53:09 +0100 Subject: [PATCH 28/67] PS: Apply formating with clang-format From 1929c86bff1c5c99c5e695e6d04303095852a63a Mon Sep 17 00:00:00 2001 From: jmelou Date: Mon, 18 Mar 2024 17:16:50 +0100 Subject: [PATCH 29/67] [cameraInit] Allow more PS folder names --- src/software/pipeline/main_cameraInit.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/software/pipeline/main_cameraInit.cpp b/src/software/pipeline/main_cameraInit.cpp index 6d1cf48278..980dbd853a 100644 --- a/src/software/pipeline/main_cameraInit.cpp +++ b/src/software/pipeline/main_cameraInit.cpp @@ -468,7 +468,10 @@ int aliceVision_main(int argc, char** argv) } } - if (boost::algorithm::starts_with(parentPath.stem().string(), "ps_") || boost::algorithm::starts_with(parentPath.stem().string(), "hdr_")) + std::string toCompare = parentPath.stem().string(); + transform(toCompare.begin(), toCompare.end(), toCompare.begin(), ::tolower); + + if (boost::algorithm::starts_with(toCompare, "ps_") || boost::algorithm::starts_with(parentPath.stem().string(), "hdr_")) { std::hash hash; IndexT tmpPoseID = hash(parentPath.string()); // use a temporary pose Id to group the images From 1373a342f8f0863cca9952740014053da1079677 Mon Sep 17 00:00:00 2001 From: jmelou Date: Tue, 19 Mar 2024 15:41:35 +0100 Subject: [PATCH 30/67] [PS] Add png export for albedo --- src/aliceVision/photometricStereo/photometricDataIO.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/aliceVision/photometricStereo/photometricDataIO.cpp b/src/aliceVision/photometricStereo/photometricDataIO.cpp index 3df7bfb9f3..e97d0f5352 100644 --- a/src/aliceVision/photometricStereo/photometricDataIO.cpp +++ b/src/aliceVision/photometricStereo/photometricDataIO.cpp @@ -522,9 +522,9 @@ void writePSResults(const std::string& outputPath, const image::Image Date: Tue, 19 Mar 2024 15:45:47 +0100 Subject: [PATCH 31/67] [PS] Data IO : change light normalisation --- .../photometricStereo/photometricDataIO.cpp | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/aliceVision/photometricStereo/photometricDataIO.cpp b/src/aliceVision/photometricStereo/photometricDataIO.cpp index e97d0f5352..3f5ff7ceab 100644 --- a/src/aliceVision/photometricStereo/photometricDataIO.cpp +++ b/src/aliceVision/photometricStereo/photometricDataIO.cpp @@ -217,11 +217,7 @@ void buildLightMatFromJSON(const std::string& fileName, } else if (cpt == 4) { - // no support for SH lighting : - for (int pictureIndex = 0; pictureIndex < lightMat.rows(); ++pictureIndex) - { - lightMat.row(pictureIndex) = lightMat.row(pictureIndex) / lightMat.row(pictureIndex).norm(); - } + lightMat.row(lineNumber) = lightMat.row(lineNumber)/lightMat.row(lineNumber).norm(); ALICEVISION_LOG_INFO("SH will soon be available for use in PS. For now, lighting is reduced to directional"); } ++cpt; @@ -252,10 +248,7 @@ void buildLightMatFromJSON(const std::string& fileName, else if (cpt == 4) { // no support for SH lighting : - for (int pictureIndex = 0; pictureIndex < lightMat.rows(); ++pictureIndex) - { - lightMat.row(pictureIndex) = lightMat.row(pictureIndex) / lightMat.row(pictureIndex).norm(); - } + lightMat.row(lineNumber) = lightMat.row(lineNumber)/lightMat.row(lineNumber).norm(); ALICEVISION_LOG_INFO("SH will soon be available for use in PS. For now, lighting is reduced to directional"); } ++cpt; From 2f9b3d28825e63560034bb708d2bf588e344075b Mon Sep 17 00:00:00 2001 From: jmelou Date: Wed, 20 Mar 2024 14:29:07 +0100 Subject: [PATCH 32/67] [LightCalibration] Add functions for elliptical case --- .../lightingEstimation/ellipseGeometry.cpp | 225 ++++++++++++++++++ .../lightingEstimation/ellipseGeometry.hpp | 71 ++++++ 2 files changed, 296 insertions(+) create mode 100644 src/aliceVision/lightingEstimation/ellipseGeometry.cpp create mode 100644 src/aliceVision/lightingEstimation/ellipseGeometry.hpp diff --git a/src/aliceVision/lightingEstimation/ellipseGeometry.cpp b/src/aliceVision/lightingEstimation/ellipseGeometry.cpp new file mode 100644 index 0000000000..0667b4c3ac --- /dev/null +++ b/src/aliceVision/lightingEstimation/ellipseGeometry.cpp @@ -0,0 +1,225 @@ +// This file is part of the AliceVision project. +// Copyright (c) 2023 AliceVision contributors. +// This Source Code Form is subject to the terms of the Mozilla Public License, +// v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +#include "ellipseGeometry.hpp" +#include +#include + +#include +#include + +#include +#include + +namespace aliceVision { +namespace lightingEstimation { + +#define PI 3.14159265 + +void quadraticFromEllipseParameters(const std::array& ellipseParameters, + Eigen::Matrix3f& Q) +{ + float phi = ellipseParameters[0]*PI/180.0; + float c_x = ellipseParameters[1]; + float c_y = ellipseParameters[2]; + float a = ellipseParameters[3]; + float b = ellipseParameters[4]; + + float test = b; + b = a; + a = test; + + float A = a*a*sin(phi)*sin(phi) + b*b*cos(phi)*cos(phi); + float B = 2*(b*b-a*a)*sin(phi)*cos(phi); + float C = a*a*cos(phi)*cos(phi) + b*b*sin(phi)*sin(phi); + float D = -2*A*c_x - B*c_y; + float E = -B*c_x - 2*C*c_y; + float F = A*c_x*c_x + C*c_y*c_y - a*a*b*b + B*c_x*c_y; + + Q << A, B/2, D/2, + B/2, C, E/2, + D/2, E/2, F; + + Q /= Q.norm(); +} + +int findUniqueIndex(const std::vector& vec) { + + std::unordered_map countMap; + + // Compter le nombre d'occurrences de chaque élément dans le vecteur + for (int num : vec) { + countMap[num]++; + } + + int toReturn = -1; + // Trouver l'élément avec une seule occurrence + for (size_t i = 0; i < vec.size(); ++i) { + if (countMap[vec[i]] == 1) { + toReturn = i; + } + } + + return toReturn; +} + +void estimateSphereCenter(const std::array& ellipseParameters, + const float sphereRadius, + const Eigen::Matrix3f& K, + std::array& sphereCenter) +{ + Eigen::Matrix3f Q; + quadraticFromEllipseParameters(ellipseParameters, Q); + + Eigen::Matrix3f M = K.transpose()*Q*K; + Eigen::EigenSolver es(M); + + Eigen::Vector3f eigval = es.eigenvalues().real(); + + std::vector eigvalSign; + + for (int i = 0; i < 3; ++i) { + eigvalSign.push_back((eigval[i] > 0) ? 1 : -1); + } + + int index = findUniqueIndex(eigvalSign); + float uniqueEigval = eigval[index]; + + float dist = sqrt(1 - ((eigval[0] + eigval[1] + eigval[2] - uniqueEigval)/2) / uniqueEigval); + + Eigen::Vector3f eigvect = es.eigenvectors().col(index).real(); + + float norm = eigvect.norm(); + float C_factor = sphereRadius*dist*eigvalSign.at(index)/norm; + Eigen::Vector3f C = C_factor*eigvect; + sphereCenter[0]=C[0]; + sphereCenter[1]=C[1]; + sphereCenter[2]=C[2]; +} + +void sphereRayIntersection(const Eigen::Vector3f& direction, + const std::array& sphereCenter, + const float sphereRadius, + float& delta, + Eigen::Vector3f& normal) +{ + float a = direction.dot(direction); + + Eigen::Vector3f spCenter; + spCenter << sphereCenter[0], sphereCenter[1], sphereCenter[2]; + + float b = 2*direction.dot(spCenter); + float c = spCenter.dot(spCenter) - sphereRadius*sphereRadius; + + delta = b*b - 4*a*c; + + float factor; + delta = std::max(delta, std::numeric_limits::epsilon()); + + if (delta >= 0) { + factor = (-b - sqrt(delta))/(2*a); + normal = -direction*factor - spCenter; + normal.normalize(); + } + else { + delta = -1; + } +} + +void estimateSphereNormals(const std::array& sphereCenter, + const float sphereRadius, + const image::Image& ellipseMask, + const Eigen::Matrix3f& K, + image::Image& normals, + image::Image& newMask) +{ + Eigen::Matrix3f invK = K.inverse(); + bool print = false; + + for (int i = 1; i < ellipseMask.rows(); ++i) { + for (int j = 1; j < ellipseMask.cols(); ++j) { + if (ellipseMask(i, j) == 1) { + // Get homogeneous coordinates of the pixel : + Eigen::Vector3f coordinates = Eigen::Vector3f(j, i, 1.0f); + + // Get the direction of the ray : + Eigen::Vector3f direction = invK*coordinates; + + // Estimate the interception of the ray with the sphere : + float delta = 0.0; + Eigen::Vector3f normal = Eigen::Vector3f(0.0f, 0.0f, 0.0f); + + print = false; + + sphereRayIntersection(direction, sphereCenter, sphereRadius, delta, normal); + + // If the ray intercepts the sphere we add this pixel to the new mask : + if(delta > 0) + { + newMask(i, j) = 1.0; + normals(i, j) = image::RGBfColor(normal[0], normal[1], normal[2]); + } + else + { + newMask(i, j) = 0.0; + normals(i, j) = image::RGBfColor(0.0, 0.0, 0.0); + } + } + } + } +} + +void getRealNormalOnSphere(const std::string& maskPath, + const Eigen::Matrix3f& K, + const float sphereRadius, + image::Image& normals, + image::Image& newMask) +{ + // Charger l'image depuis le chemin spécifié + cv::Mat img = cv::imread(maskPath); + + // Convertir l'image en niveaux de gris + cv::Mat gray; + cv::cvtColor(img, gray, cv::COLOR_BGR2GRAY); + + // Appliquer un seuillage pour convertir l'image en noir et blanc + int thresholdValue = 150; + cv::Mat thresh; + cv::threshold(gray, thresh, thresholdValue, 255, 0); + + // Trouver les contours + std::vector> contours; + std::vector hierarchy; + cv::findContours(thresh, contours, hierarchy, cv::RETR_TREE, cv::CHAIN_APPROX_SIMPLE); + + // Sélectionner le premier contour + std::vector cnt = contours[0]; + + // Ajuster l'ellipse sur l'objet sélectionné + cv::RotatedRect ellipse = cv::fitEllipse(cnt); + float angle = ellipse.angle; + cv::Size2f size = ellipse.size; + cv::Point2f center = ellipse.center; + + // Convertir l'ellipse en un tableau de 5 paramètres: + std::array ellipseParameters; + + ellipseParameters[0] = angle; + ellipseParameters[1] = center.x; + ellipseParameters[2] = center.y; + ellipseParameters[3] = size.width/2; + ellipseParameters[4] = size.height/2; + + std::array sphereCenter; + estimateSphereCenter(ellipseParameters, sphereRadius, K, sphereCenter); + + image::Image mask; + photometricStereo::loadMask(maskPath, mask); + estimateSphereNormals(sphereCenter, sphereRadius, mask, K, normals, newMask); +} + +} // namespace lightingEstimation +} // namespace aliceVision \ No newline at end of file diff --git a/src/aliceVision/lightingEstimation/ellipseGeometry.hpp b/src/aliceVision/lightingEstimation/ellipseGeometry.hpp new file mode 100644 index 0000000000..34c9c0a08c --- /dev/null +++ b/src/aliceVision/lightingEstimation/ellipseGeometry.hpp @@ -0,0 +1,71 @@ +// This file is part of the AliceVision project. +// Copyright (c) 2024 AliceVision contributors. +// This Source Code Form is subject to the terms of the Mozilla Public License, +// v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +#pragma once + +#include + +#include +#include +#include + +namespace aliceVision { +namespace lightingEstimation { + +/** + * @brief Estimate the center of a sphere from the ellipse parameters + * @param[in] ellipseParameters An array of 5 floating-point: the parameters of the ellipse + * @param[in] sphereRadius The radius of the sphere + * @param[in] K Intrinsic parameters of the camera + * @param[out] sphereCenter An array of 3 floating-point: the coordinates of the sphere center in the picture frame + */ +void estimateSphereCenter(const std::array& ellipseParameters, + const float sphereRadius, + const Eigen::Matrix3f& K, + std::array& sphereCenter); + +void sphereRayIntersection(const Eigen::Vector3f& direction, + const std::array& sphereCenter, + const float sphereRadius, + float& delta, + Eigen::Vector3f& normal); + +void quadraticFromEllipseParameters(const std::array& ellipseParameters, + Eigen::Matrix3f& Q); + +int findUniqueIndex(const std::vector& vec); + +/** + * @brief Estimate the normals of a sphere from a mask + * @param[in] sphereCenter An array of 3 floating-point: the coordinates of the sphere center in the picture frame + * @param[in] ellipseMask The binary mask of the sphere in the picture + * @param[in] K Intrinsic parameters of the camera + * @param[out] normals Normals on the sphere + * @param[out] newMask The mask of the sphere after ray tracing + */ +void estimateSphereNormals(const std::array& sphereCenter, + const float sphereRadius, + const image::Image& ellipseMask, + const Eigen::Matrix3f& K, + image::Image& normals, + image::Image& newMask); + +/** + * @brief Estimate the normals of a sphere from a mask + * @param[in] maskPath The path to the binary mask of the ellipse in the picture + * @param[in] K Intrinsic parameters of the camera + * @param[out] normals Normals on the sphere in camera frame + * @param[out] newMask The mask of the sphere after ray tracing +*/ +void getRealNormalOnSphere(const std::string& maskPath, + const Eigen::Matrix3f& K, + const float sphereRadius, + image::Image& normals, + image::Image& newMask); + + +} // namespace lightingEstimation +} // namespace aliceVision \ No newline at end of file From 8514055149f22af63d5b1c0213c2a2bf9d9ed700 Mon Sep 17 00:00:00 2001 From: jmelou Date: Wed, 20 Mar 2024 17:01:59 +0100 Subject: [PATCH 33/67] [PS] LightCalibration : allow use of mask instead of json --- .../lightingCalibration.cpp | 119 +++++++++++++----- 1 file changed, 87 insertions(+), 32 deletions(-) diff --git a/src/aliceVision/lightingEstimation/lightingCalibration.cpp b/src/aliceVision/lightingEstimation/lightingCalibration.cpp index 262f644ce3..ac162b6026 100644 --- a/src/aliceVision/lightingEstimation/lightingCalibration.cpp +++ b/src/aliceVision/lightingEstimation/lightingCalibration.cpp @@ -40,16 +40,8 @@ void lightCalibration(const sfmData::SfMData& sfmData, const bool saveAsModel) { std::vector imageList; - std::vector> allSpheresParams; std::vector focals; - std::string inputJSONFullName = inputJSON + "/detection.json"; - - // Main tree - bpt::ptree fileTree; - // Read the json file and initialize the tree - bpt::read_json(inputJSONFullName, fileTree); - std::map viewMap; for (auto& viewIt : sfmData.getViews()) { @@ -66,36 +58,89 @@ void lightCalibration(const sfmData::SfMData& sfmData, } } - for (const auto& [currentTime, currentView] : viewMap) + bool fromJSON = false; + + std::vector> allSpheresParams; + std::vector KMatrices; + + std::string inputJSONFullName = inputJSON + "/detection.json"; + std::string maskFullName = inputJSON + "/mask.png"; + + if(fs::exists(inputJSONFullName)) + { + std::cout << "JSON file detected" << std::endl; + fromJSON = true; + } + + if (fromJSON) { - ALICEVISION_LOG_INFO("View Id: " << currentView.getViewId()); - const fs::path imagePath = fs::path(currentView.getImage().getImagePath()); + // Main tree + bpt::ptree fileTree; + // Read the json file and initialize the tree + bpt::read_json(inputJSONFullName, fileTree); - if (!boost::algorithm::icontains(imagePath.stem().string(), "ambiant")) + for (const auto& [currentTime, currentView] : viewMap) { - std::string sphereName = std::to_string(currentView.getViewId()); - auto sphereExists = (fileTree.get_child_optional(sphereName)).is_initialized(); - if (sphereExists) - { - ALICEVISION_LOG_INFO(" - " << imagePath.string()); - imageList.push_back(imagePath.string()); + ALICEVISION_LOG_INFO("View Id: " << currentView.getViewId()); + const fs::path imagePath = fs::path(currentView.getImage().getImagePath()); - std::array currentSphereParams; - for (auto& currentSphere : fileTree.get_child(sphereName)) + if (!boost::algorithm::icontains(imagePath.stem().string(), "ambiant")) + { + std::string sphereName = std::to_string(currentView.getViewId()); + auto sphereExists = (fileTree.get_child_optional(sphereName)).is_initialized(); + if (sphereExists) { - currentSphereParams[0] = currentSphere.second.get_child("").get("x", 0.0); - currentSphereParams[1] = currentSphere.second.get_child("").get("y", 0.0); - currentSphereParams[2] = currentSphere.second.get_child("").get("r", 0.0); - } + ALICEVISION_LOG_INFO(" - " << imagePath.string()); + imageList.push_back(imagePath.string()); + + std::array currentSphereParams; + for (auto& currentSphere : fileTree.get_child(sphereName)) + { + currentSphereParams[0] = currentSphere.second.get_child("").get("x", 0.0); + currentSphereParams[1] = currentSphere.second.get_child("").get("y", 0.0); + currentSphereParams[2] = currentSphere.second.get_child("").get("r", 0.0); + } - allSpheresParams.push_back(currentSphereParams); + allSpheresParams.push_back(currentSphereParams); - IndexT intrinsicId = currentView.getIntrinsicId(); - focals.push_back(sfmData.getIntrinsics().at(intrinsicId)->getParams().at(0)); + IndexT intrinsicId = currentView.getIntrinsicId(); + focals.push_back(sfmData.getIntrinsics().at(intrinsicId)->getParams().at(0)); + } + else + { + ALICEVISION_LOG_WARNING("No detected sphere found for '" << imagePath << "'."); + } } - else + } + } + else + { + IndexT viewId; + for (const auto& [currentTime, currentView] : viewMap) + { + ALICEVISION_LOG_INFO("View Id: " << currentView.getViewId()); + const fs::path imagePath = fs::path(currentView.getImage().getImagePath()); + + if (!boost::algorithm::icontains(imagePath.stem().string(), "ambiant")) { - ALICEVISION_LOG_WARNING("No detected sphere found for '" << imagePath << "'."); + ALICEVISION_LOG_INFO(" - " << imagePath.string()); + imageList.push_back(imagePath.string()); + viewId = currentView.getViewId(); + // Get intrinsics associated with this view : + IndexT intrinsicId = currentView.getIntrinsicId(); + const float focalPx = sfmData.getIntrinsics().at(intrinsicId)->getParams().at(0); + int nbCols = sfmData.getIntrinsics().at(intrinsicId)->w(); + int nbRows = sfmData.getIntrinsics().at(intrinsicId)->h(); + const float x_p = (nbCols) / 2 + sfmData.getIntrinsics().at(intrinsicId)->getParams().at(2); + const float y_p = (nbRows) / 2 + sfmData.getIntrinsics().at(intrinsicId)->getParams().at(3); + + Eigen::MatrixXf currentK = Eigen::MatrixXf::Zero(3, 3); + // Create K matrix + currentK << focalPx, 0.0, x_p, + 0.0, focalPx, y_p, + 0.0, 0.0, 1.0; + + KMatrices.push_back(currentK); } } } @@ -109,13 +154,23 @@ void lightCalibration(const sfmData::SfMData& sfmData, for (size_t i = 0; i < imageList.size(); ++i) { + std::string picturePath = imageList.at(i); - std::array sphereParam = allSpheresParams.at(i); - float focal = focals.at(i); Eigen::VectorXf lightingDirection = Eigen::VectorXf::Zero(lightSize); float intensity; - lightCalibrationOneImage(picturePath, sphereParam, focal, method, lightingDirection, intensity); + if(fromJSON) + { + float focal = focals.at(i); + std::array sphereParam = allSpheresParams.at(i); + lightCalibrationOneImage(picturePath, sphereParam, focal, method, lightingDirection, intensity); + } + else + { + Eigen::Matrix3f K = KMatrices.at(i); + float sphereRadius = 1.0; + calibrateLightFromRealSphere(picturePath, maskFullName, K, sphereRadius, method, lightingDirection, intensity); + } lightMat.row(i) = lightingDirection; intList.push_back(intensity); From 8a3c0d72b98b93c28740808e4e73ace4f74d0256 Mon Sep 17 00:00:00 2001 From: jmelou Date: Wed, 20 Mar 2024 17:03:52 +0100 Subject: [PATCH 34/67] [PS] Lighting Calibration : add functions for calibration from mask --- .../lightingCalibration.cpp | 105 ++++++++++++++++++ .../lightingCalibration.hpp | 15 +++ 2 files changed, 120 insertions(+) diff --git a/src/aliceVision/lightingEstimation/lightingCalibration.cpp b/src/aliceVision/lightingEstimation/lightingCalibration.cpp index ac162b6026..eb5d4affba 100644 --- a/src/aliceVision/lightingEstimation/lightingCalibration.cpp +++ b/src/aliceVision/lightingEstimation/lightingCalibration.cpp @@ -340,6 +340,52 @@ void lightCalibrationOneImage(const std::string& picturePath, lightingDirection << directionnalPart, secondOrder; } +void calibrateLightFromRealSphere(const std::string& picturePath, + const std::string& maskPath, + const Eigen::Matrix3f& K, + const float sphereRadius, + const std::string& method, + Eigen::VectorXf& lightingDirection, + float& intensity) +{ + // Read picture : + image::Image imageFloat; + image::readImage(picturePath, imageFloat, image::EImageColorSpace::NO_CONVERSION); + + image::Image normals(imageFloat.width(), imageFloat.height()); + image::Image newMask(imageFloat.width(), imageFloat.height()); + + getRealNormalOnSphere(maskPath, K, sphereRadius, normals, newMask); + + // If method = brightest point : + if (!method.compare("brightestPoint")) + { + // Detect brightest point : + Eigen::Vector2f brigthestPoint; + detectBrightestPoint(newMask, imageFloat, brigthestPoint); + + Eigen::Vector2f brigthestPoint_xy; + brigthestPoint_xy(0) = brigthestPoint(0) - imageFloat.cols() / 2; + brigthestPoint_xy(1) = brigthestPoint(1) - imageFloat.rows() / 2; + + Eigen::Vector3f normalBrightestPoint; + normalBrightestPoint = normals(round(brigthestPoint(1)), round(brigthestPoint(0))).cast(); + + // Observation direction : + Eigen::Vector3f observationRayPersp; + + // orthographic approximation : + observationRayPersp(0) = brigthestPoint_xy(0) / K(0,0); + observationRayPersp(1) = brigthestPoint_xy(1) / K(0,0); + observationRayPersp(2) = 1.0; + observationRayPersp = -observationRayPersp / observationRayPersp.norm(); + + // Evaluate lighting direction : + lightingDirection = 2 * normalBrightestPoint.dot(observationRayPersp) * normalBrightestPoint - observationRayPersp; + lightingDirection = lightingDirection / lightingDirection.norm(); + + intensity = 1.0; + } } void detectBrightestPoint(const std::array& sphereParam, const image::Image& imageFloat, Eigen::Vector2f& brigthestPoint) @@ -364,6 +410,28 @@ void detectBrightestPoint(const std::array& sphereParam, const image:: brigthestPoint(0) = maxCol + patchOrigin[0] - imageFloat.cols() / 2; brigthestPoint(1) = maxRow + patchOrigin[1] - imageFloat.rows() / 2; +void detectBrightestPoint(const image::Image newMask, const image::Image& imageFloat, Eigen::Vector2f& brigthestPoint) +{ + image::Image patch; + std::array patchOrigin; + cutImage(imageFloat, newMask, patch, patchOrigin); + + image::Image convolutedPatch1; + image::Image convolutedPatch2; + + // Create Kernel + size_t kernelSize = round(patch.rows() / 40); // arbitrary + Eigen::VectorXf kernel(2 * kernelSize + 1); + createTriangleKernel(kernelSize, kernel); + + image::imageVerticalConvolution(patch, kernel, convolutedPatch1); + image::imageHorizontalConvolution(convolutedPatch1, kernel, convolutedPatch2); + + Eigen::Index maxRow, maxCol; + static_cast(convolutedPatch2.maxCoeff(&maxRow, &maxCol)); + + brigthestPoint(0) = maxCol + patchOrigin[0]; + brigthestPoint(1) = maxRow + patchOrigin[1]; } void createTriangleKernel(const size_t kernelSize, Eigen::VectorXf& kernel) @@ -416,6 +484,43 @@ void cutImage(const image::Image& imageFloat, } } +void cutImage(const image::Image& imageFloat, + const image::Image& newMask, + image::Image& patch, + std::array& patchOrigin) +{ + int minISphere = newMask.rows(); + int minJSphere = newMask.cols(); + int maxISphere = 0; + int maxJSphere = 0; + + for (int j = 0; j < newMask.cols(); ++j) + { + for (int i = 0; i < newMask.rows(); ++i) + { + if (newMask(i, j) == 1) + { + if(minISphere > i) + minISphere = i; + + if(minJSphere > j) + minJSphere = j; + + if(maxISphere < i) + maxISphere = i; + + if(maxJSphere < j) + maxJSphere = j; + } + } + } + + patchOrigin[0] = minJSphere; + patchOrigin[1] = minISphere; + + patch = imageFloat.block(minISphere, minJSphere, maxISphere-minISphere, maxJSphere-minJSphere); +} + void writeJSON(const std::string& fileName, const sfmData::SfMData& sfmData, const std::vector& imageList, diff --git a/src/aliceVision/lightingEstimation/lightingCalibration.hpp b/src/aliceVision/lightingEstimation/lightingCalibration.hpp index 3fdb7369da..e438d2914e 100644 --- a/src/aliceVision/lightingEstimation/lightingCalibration.hpp +++ b/src/aliceVision/lightingEstimation/lightingCalibration.hpp @@ -43,6 +43,14 @@ void lightCalibrationOneImage(const std::string& picturePath, Eigen::VectorXf& lightingDirection, float& intensity); +void calibrateLightFromRealSphere(const std::string& picturePath, + const std::string& maskPath, + const Eigen::Matrix3f& K, + const float sphereRadius, + const std::string& method, + Eigen::VectorXf& lightingDirection, + float& intensity); + /** * @brief Compute the brightest point on a sphere * This function cuts the input image around the sphere and applies a convolution filter to find the brightest point in the cut region @@ -52,6 +60,8 @@ void lightCalibrationOneImage(const std::string& picturePath, */ void detectBrightestPoint(const std::array& sphereParam, const image::Image& imageFloat, Eigen::Vector2f& brigthestPoint); +void detectBrightestPoint(const image::Image newMask, const image::Image& imageFloat, Eigen::Vector2f& brigthestPoint); + /** * @brief Create a triangle kernel * @param[in] kernelSize Size of the kernel (should be an odd number) @@ -80,6 +90,11 @@ void cutImage(const image::Image& imageFloat, image::Image& patch, std::array& patchOrigin); +void cutImage(const image::Image& imageFloat, + const image::Image& newMask, + image::Image& patch, + std::array& patchOrigin); + /** * @brief Write a JSON file containing light information * @param[in] fileName The path to the JSON file to generate From d32330462c9597c8617410fff8259daeedf278da Mon Sep 17 00:00:00 2001 From: jmelou Date: Wed, 20 Mar 2024 17:04:52 +0100 Subject: [PATCH 35/67] [PS] Lighting calibration : debug perspective estimation --- .../lightingCalibration.cpp | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/aliceVision/lightingEstimation/lightingCalibration.cpp b/src/aliceVision/lightingEstimation/lightingCalibration.cpp index eb5d4affba..4c7140b206 100644 --- a/src/aliceVision/lightingEstimation/lightingCalibration.cpp +++ b/src/aliceVision/lightingEstimation/lightingCalibration.cpp @@ -198,15 +198,23 @@ void lightCalibrationOneImage(const std::string& picturePath, Eigen::Vector2f brigthestPoint; detectBrightestPoint(sphereParam, imageFloat, brigthestPoint); + Eigen::Vector2f brigthestPoint_xy; + brigthestPoint_xy(0) = brigthestPoint(0) - imageFloat.cols() / 2; + brigthestPoint_xy(1) = brigthestPoint(1) - imageFloat.rows() / 2; + Eigen::Vector3f normalBrightestPoint; - getNormalOnSphere(brigthestPoint(0), brigthestPoint(1), sphereParam, normalBrightestPoint); + getNormalOnSphere(brigthestPoint_xy(0), brigthestPoint_xy(1), sphereParam, normalBrightestPoint); // Observation direction : Eigen::Vector3f observationRayPersp; - observationRayPersp(0) = (brigthestPoint(0) - imageFloat.cols() / 2) / focal; - observationRayPersp(1) = (brigthestPoint(1) - imageFloat.rows() / 2) / focal; - observationRayPersp(2) = 1; + // orthographic approximation : + //observationRay(0) = 0.0; + //observationRay(1) = 0.0; + //observationRay(2) = -1.0; + observationRayPersp(0) = brigthestPoint_xy(0) / focal; + observationRayPersp(1) = brigthestPoint_xy(1) / focal; + observationRayPersp(2) = 1.0; observationRayPersp = -observationRayPersp / observationRayPersp.norm(); // Evaluate lighting direction : @@ -408,8 +416,10 @@ void detectBrightestPoint(const std::array& sphereParam, const image:: Eigen::Index maxRow, maxCol; static_cast(convolutedPatch2.maxCoeff(&maxRow, &maxCol)); - brigthestPoint(0) = maxCol + patchOrigin[0] - imageFloat.cols() / 2; - brigthestPoint(1) = maxRow + patchOrigin[1] - imageFloat.rows() / 2; + brigthestPoint(0) = maxCol + patchOrigin[0]; + brigthestPoint(1) = maxRow + patchOrigin[1]; +} + void detectBrightestPoint(const image::Image newMask, const image::Image& imageFloat, Eigen::Vector2f& brigthestPoint) { image::Image patch; From ff4bb7b5c07fdbfc52dd9989c408681a466fd8fb Mon Sep 17 00:00:00 2001 From: jmelou Date: Wed, 20 Mar 2024 17:05:15 +0100 Subject: [PATCH 36/67] [PS] Lighting calibration : add needed include --- src/aliceVision/lightingEstimation/lightingCalibration.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/aliceVision/lightingEstimation/lightingCalibration.cpp b/src/aliceVision/lightingEstimation/lightingCalibration.cpp index 4c7140b206..74bc579543 100644 --- a/src/aliceVision/lightingEstimation/lightingCalibration.cpp +++ b/src/aliceVision/lightingEstimation/lightingCalibration.cpp @@ -6,6 +6,7 @@ #include "lightingCalibration.hpp" #include "lightingEstimation.hpp" +#include "ellipseGeometry.hpp" #include #include From cdf234f799318fd6650c752a3b99f26632c2a744 Mon Sep 17 00:00:00 2001 From: jmelou Date: Wed, 20 Mar 2024 17:05:48 +0100 Subject: [PATCH 37/67] [PS] Lighting calibration : allows use of mask instead of json --- src/software/pipeline/main_lightingCalibration.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/software/pipeline/main_lightingCalibration.cpp b/src/software/pipeline/main_lightingCalibration.cpp index 3c4a5c1a76..cce465b1dc 100644 --- a/src/software/pipeline/main_lightingCalibration.cpp +++ b/src/software/pipeline/main_lightingCalibration.cpp @@ -13,6 +13,7 @@ // Lighting calibration #include +#include // Command line parameters #include @@ -47,7 +48,7 @@ int aliceVision_main(int argc, char** argv) system::Timer timer; std::string inputPath; - std::string inputJSON; + std::string inputDetection; std::string ouputJSON; std::string method; bool saveAsModel; @@ -57,8 +58,8 @@ int aliceVision_main(int argc, char** argv) requiredParams.add_options() ("inputPath,i", po::value(&inputPath)->required(), "Path to the SfMData input.") - ("inputJSON, j", po::value(&inputJSON)->required(), - "Path to the folder containing the JSON file that describes spheres' positions and radius.") + ("inputDetection, j", po::value(&inputDetection)->required(), + "Path to the folder containing the JSON file that describes spheres' positions and radius") ("outputFile, o", po::value(&ouputJSON)->required(), "Path to JSON output file."); @@ -92,7 +93,8 @@ int aliceVision_main(int argc, char** argv) ALICEVISION_LOG_ERROR("The input file '" + inputPath + "' cannot be read."); return EXIT_FAILURE; } - lightingEstimation::lightCalibration(sfmData, inputJSON, ouputJSON, method, saveAsModel); + + lightingEstimation::lightCalibration(sfmData, inputDetection, ouputJSON, method, saveAsModel); } ALICEVISION_LOG_INFO("Task done in (s): " + std::to_string(timer.elapsed())); From 8da0dc92dbc64c2f7ab33a2dff56f3953be83f05 Mon Sep 17 00:00:00 2001 From: jmelou Date: Thu, 21 Mar 2024 13:46:56 +0100 Subject: [PATCH 38/67] [PS] Lighting calibration : sphere detection file as input instead of folder --- .../lightingEstimation/lightingCalibration.cpp | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/aliceVision/lightingEstimation/lightingCalibration.cpp b/src/aliceVision/lightingEstimation/lightingCalibration.cpp index 74bc579543..209e353361 100644 --- a/src/aliceVision/lightingEstimation/lightingCalibration.cpp +++ b/src/aliceVision/lightingEstimation/lightingCalibration.cpp @@ -35,7 +35,7 @@ namespace aliceVision { namespace lightingEstimation { void lightCalibration(const sfmData::SfMData& sfmData, - const std::string& inputJSON, + const std::string& inputFile, const std::string& outputPath, const std::string& method, const bool saveAsModel) @@ -64,10 +64,7 @@ void lightCalibration(const sfmData::SfMData& sfmData, std::vector> allSpheresParams; std::vector KMatrices; - std::string inputJSONFullName = inputJSON + "/detection.json"; - std::string maskFullName = inputJSON + "/mask.png"; - - if(fs::exists(inputJSONFullName)) + if(fs::path(inputFile).extension() == ".json") { std::cout << "JSON file detected" << std::endl; fromJSON = true; @@ -78,7 +75,7 @@ void lightCalibration(const sfmData::SfMData& sfmData, // Main tree bpt::ptree fileTree; // Read the json file and initialize the tree - bpt::read_json(inputJSONFullName, fileTree); + bpt::read_json(inputFile, fileTree); for (const auto& [currentTime, currentView] : viewMap) { @@ -170,7 +167,7 @@ void lightCalibration(const sfmData::SfMData& sfmData, { Eigen::Matrix3f K = KMatrices.at(i); float sphereRadius = 1.0; - calibrateLightFromRealSphere(picturePath, maskFullName, K, sphereRadius, method, lightingDirection, intensity); + calibrateLightFromRealSphere(picturePath, inputFile, K, sphereRadius, method, lightingDirection, intensity); } lightMat.row(i) = lightingDirection; From b1847aa87426114b8287fb6ccbc82af5fa65a496 Mon Sep 17 00:00:00 2001 From: jmelou Date: Mon, 29 Apr 2024 11:29:44 +0200 Subject: [PATCH 39/67] [PS] Fix png output --- src/aliceVision/photometricStereo/photometricDataIO.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/aliceVision/photometricStereo/photometricDataIO.cpp b/src/aliceVision/photometricStereo/photometricDataIO.cpp index 3f5ff7ceab..80ee00e52b 100644 --- a/src/aliceVision/photometricStereo/photometricDataIO.cpp +++ b/src/aliceVision/photometricStereo/photometricDataIO.cpp @@ -469,9 +469,9 @@ void convertNormalMap2png(const image::Image& normalsIm, image } else { - normalsImPNG(i, j)(0) = floor(255 * (normalsIm(i, j)(0) + 1) / 2); - normalsImPNG(i, j)(1) = -floor(255 * (normalsIm(i, j)(1) + 1) / 2); - normalsImPNG(i, j)(2) = -floor(255 * normalsIm(i, j)(2)); + normalsImPNG(i, j)(0) = floor(255.0 * (normalsIm(i, j)(0) + 1.0) / 2.0); + normalsImPNG(i, j)(1) = -ceil(255.0 * (normalsIm(i, j)(1) + 1.0) / 2.0); + normalsImPNG(i, j)(2) = -ceil(255.0 * normalsIm(i, j)(2)); } } } From 8bbe1275c9c64450955ea1eeae52060d9734cff6 Mon Sep 17 00:00:00 2001 From: jmelou Date: Mon, 29 Apr 2024 11:43:46 +0200 Subject: [PATCH 40/67] [lightingCalibration] Debug ellipse geometry --- .../lightingEstimation/ellipseGeometry.cpp | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/aliceVision/lightingEstimation/ellipseGeometry.cpp b/src/aliceVision/lightingEstimation/ellipseGeometry.cpp index 0667b4c3ac..7528027d2d 100644 --- a/src/aliceVision/lightingEstimation/ellipseGeometry.cpp +++ b/src/aliceVision/lightingEstimation/ellipseGeometry.cpp @@ -25,12 +25,8 @@ void quadraticFromEllipseParameters(const std::array& ellipseParameter float phi = ellipseParameters[0]*PI/180.0; float c_x = ellipseParameters[1]; float c_y = ellipseParameters[2]; - float a = ellipseParameters[3]; - float b = ellipseParameters[4]; - - float test = b; - b = a; - a = test; + float b = ellipseParameters[3]; + float a = ellipseParameters[4]; float A = a*a*sin(phi)*sin(phi) + b*b*cos(phi)*cos(phi); float B = 2*(b*b-a*a)*sin(phi)*cos(phi); @@ -91,9 +87,10 @@ void estimateSphereCenter(const std::array& ellipseParameters, float dist = sqrt(1 - ((eigval[0] + eigval[1] + eigval[2] - uniqueEigval)/2) / uniqueEigval); Eigen::Vector3f eigvect = es.eigenvectors().col(index).real(); + float sign = eigvect[2] > 0 ? 1 : -1; float norm = eigvect.norm(); - float C_factor = sphereRadius*dist*eigvalSign.at(index)/norm; + float C_factor = sphereRadius*dist*sign/norm; Eigen::Vector3f C = C_factor*eigvect; sphereCenter[0]=C[0]; sphereCenter[1]=C[1]; @@ -111,18 +108,16 @@ void sphereRayIntersection(const Eigen::Vector3f& direction, Eigen::Vector3f spCenter; spCenter << sphereCenter[0], sphereCenter[1], sphereCenter[2]; - float b = 2*direction.dot(spCenter); + float b = -2*direction.dot(spCenter); float c = spCenter.dot(spCenter) - sphereRadius*sphereRadius; delta = b*b - 4*a*c; float factor; - delta = std::max(delta, std::numeric_limits::epsilon()); if (delta >= 0) { factor = (-b - sqrt(delta))/(2*a); - normal = -direction*factor - spCenter; - normal.normalize(); + normal = (direction*factor - spCenter)/sphereRadius; } else { delta = -1; From 1c65593c76d42cdc1b6be359b0c6260101a0abe1 Mon Sep 17 00:00:00 2001 From: jmelou Date: Mon, 29 Apr 2024 11:44:00 +0200 Subject: [PATCH 41/67] [sphereDetection] Change output --- src/aliceVision/sphereDetection/sphereDetection.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aliceVision/sphereDetection/sphereDetection.cpp b/src/aliceVision/sphereDetection/sphereDetection.cpp index 05b95ea31b..fb0c9e5d6b 100644 --- a/src/aliceVision/sphereDetection/sphereDetection.cpp +++ b/src/aliceVision/sphereDetection/sphereDetection.cpp @@ -246,7 +246,7 @@ void writeManualSphereJSON(const sfmData::SfMData& sfmData, const std::array Date: Mon, 29 Apr 2024 11:48:48 +0200 Subject: [PATCH 42/67] [lightingCalibration] Use CVmask fir ellipse --- .../lightingCalibration.cpp | 100 +++++++++++++++--- .../lightingCalibration.hpp | 4 +- 2 files changed, 90 insertions(+), 14 deletions(-) diff --git a/src/aliceVision/lightingEstimation/lightingCalibration.cpp b/src/aliceVision/lightingEstimation/lightingCalibration.cpp index 209e353361..947e8b4089 100644 --- a/src/aliceVision/lightingEstimation/lightingCalibration.cpp +++ b/src/aliceVision/lightingEstimation/lightingCalibration.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include @@ -59,6 +60,7 @@ void lightCalibration(const sfmData::SfMData& sfmData, } } + bool testEllipse = true; bool fromJSON = false; std::vector> allSpheresParams; @@ -70,6 +72,9 @@ void lightCalibration(const sfmData::SfMData& sfmData, fromJSON = true; } + int nbCols = 0; + int nbRows = 0; + if (fromJSON) { // Main tree @@ -103,6 +108,20 @@ void lightCalibration(const sfmData::SfMData& sfmData, IndexT intrinsicId = currentView.getIntrinsicId(); focals.push_back(sfmData.getIntrinsics().at(intrinsicId)->getParams().at(0)); + + float focalPx = sfmData.getIntrinsics().at(intrinsicId)->getParams().at(0); + nbCols = sfmData.getIntrinsics().at(intrinsicId)->w(); + nbRows = sfmData.getIntrinsics().at(intrinsicId)->h(); + float x_p = (nbCols) / 2 + sfmData.getIntrinsics().at(intrinsicId)->getParams().at(2); + float y_p = (nbRows) / 2 + sfmData.getIntrinsics().at(intrinsicId)->getParams().at(3); + + Eigen::MatrixXf currentK = Eigen::MatrixXf::Zero(3, 3); + // Create K matrix + currentK << focalPx, 0.0, x_p, + 0.0, focalPx, y_p, + 0.0, 0.0, 1.0; + + KMatrices.push_back(currentK); } else { @@ -127,8 +146,8 @@ void lightCalibration(const sfmData::SfMData& sfmData, // Get intrinsics associated with this view : IndexT intrinsicId = currentView.getIntrinsicId(); const float focalPx = sfmData.getIntrinsics().at(intrinsicId)->getParams().at(0); - int nbCols = sfmData.getIntrinsics().at(intrinsicId)->w(); - int nbRows = sfmData.getIntrinsics().at(intrinsicId)->h(); + nbCols = sfmData.getIntrinsics().at(intrinsicId)->w(); + nbRows = sfmData.getIntrinsics().at(intrinsicId)->h(); const float x_p = (nbCols) / 2 + sfmData.getIntrinsics().at(intrinsicId)->getParams().at(2); const float y_p = (nbRows) / 2 + sfmData.getIntrinsics().at(intrinsicId)->getParams().at(3); @@ -161,15 +180,36 @@ void lightCalibration(const sfmData::SfMData& sfmData, { float focal = focals.at(i); std::array sphereParam = allSpheresParams.at(i); - lightCalibrationOneImage(picturePath, sphereParam, focal, method, lightingDirection, intensity); + if(testEllipse) + { + Eigen::Matrix3f K = KMatrices.at(i); + float sphereRadius = 1.0; + std::array ellipseParam; + + cv::Mat maskCV = cv::Mat::zeros(nbRows, nbCols, CV_8UC1); + + image::Image imageFloat; + image::readImage(picturePath, imageFloat, image::EImageColorSpace::NO_CONVERSION); + getEllipseMaskFromSphereParameters(sphereParam, K, ellipseParam, maskCV); + calibrateLightFromRealSphere(imageFloat, maskCV, K, sphereRadius, method, lightingDirection, intensity); + } + else + { + lightCalibrationOneImage(picturePath, sphereParam, focal, method, lightingDirection, intensity); + } } else { Eigen::Matrix3f K = KMatrices.at(i); float sphereRadius = 1.0; - calibrateLightFromRealSphere(picturePath, inputFile, K, sphereRadius, method, lightingDirection, intensity); - } + cv::Mat maskCV = cv::imread(inputFile, cv::IMREAD_GRAYSCALE); + image::Image imageFloat; + image::readImage(picturePath, imageFloat, image::EImageColorSpace::NO_CONVERSION); + calibrateLightFromRealSphere(imageFloat, maskCV, K, sphereRadius, method, lightingDirection, intensity); + + break; + } lightMat.row(i) = lightingDirection; intList.push_back(intensity); } @@ -346,22 +386,21 @@ void lightCalibrationOneImage(const std::string& picturePath, lightingDirection << directionnalPart, secondOrder; } -void calibrateLightFromRealSphere(const std::string& picturePath, - const std::string& maskPath, +void calibrateLightFromRealSphere(const image::Image& imageFloat, + const cv::Mat& maskCV, const Eigen::Matrix3f& K, const float sphereRadius, const std::string& method, Eigen::VectorXf& lightingDirection, float& intensity) { - // Read picture : - image::Image imageFloat; - image::readImage(picturePath, imageFloat, image::EImageColorSpace::NO_CONVERSION); - image::Image normals(imageFloat.width(), imageFloat.height()); image::Image newMask(imageFloat.width(), imageFloat.height()); - getRealNormalOnSphere(maskPath, K, sphereRadius, normals, newMask); + getRealNormalOnSphere(maskCV, K, sphereRadius, normals, newMask); + + image::Image normalsPNG(maskCV.cols, maskCV.rows); + aliceVision::photometricStereo::convertNormalMap2png(normals, normalsPNG); // If method = brightest point : if (!method.compare("brightestPoint")) @@ -392,6 +431,43 @@ void calibrateLightFromRealSphere(const std::string& picturePath, intensity = 1.0; } + + // If method = whiteSphere : + else if (!method.compare("whiteSphere")) + { + // Evaluate light direction and intensity by pseudo-inverse + + std::vector indices; + aliceVision::photometricStereo::getIndMask(newMask, indices); + + Eigen::VectorXf imSphere(indices.size()); + Eigen::MatrixXf normalSphere(indices.size(), 3); + + int currentIndex = 0; + + for (int j = 0; j < newMask.cols(); ++j) + { + for (int i = 0; i < newMask.rows(); ++i) + { + if (newMask(i,j) > 0.1 && (imageFloat(i, j) > 0.2) && (imageFloat(i, j) < 0.8)) + { + // imSphere = normalSphere.s + imSphere(currentIndex) = imageFloat(i, j); + + normalSphere(currentIndex, 0) = normals(i,j)(0); + normalSphere(currentIndex, 1) = normals(i,j)(1); + normalSphere(currentIndex, 2) = normals(i,j)(2); + + ++currentIndex; + } + } + } + + lightingDirection = normalSphere.colPivHouseholderQr().solve(imSphere); + + intensity = lightingDirection.norm(); + lightingDirection = lightingDirection / intensity; + } } void detectBrightestPoint(const std::array& sphereParam, const image::Image& imageFloat, Eigen::Vector2f& brigthestPoint) diff --git a/src/aliceVision/lightingEstimation/lightingCalibration.hpp b/src/aliceVision/lightingEstimation/lightingCalibration.hpp index e438d2914e..c8e94f3d90 100644 --- a/src/aliceVision/lightingEstimation/lightingCalibration.hpp +++ b/src/aliceVision/lightingEstimation/lightingCalibration.hpp @@ -43,8 +43,8 @@ void lightCalibrationOneImage(const std::string& picturePath, Eigen::VectorXf& lightingDirection, float& intensity); -void calibrateLightFromRealSphere(const std::string& picturePath, - const std::string& maskPath, +void calibrateLightFromRealSphere(const image::Image& imageFloat, + const cv::Mat& maskCV, const Eigen::Matrix3f& K, const float sphereRadius, const std::string& method, From 61e98b0ca5c8ec7008f160d629aa8e04a54f5f09 Mon Sep 17 00:00:00 2001 From: jmelou Date: Mon, 29 Apr 2024 11:54:13 +0200 Subject: [PATCH 43/67] [LightingCalibration] Ellipse geometry : use OpenCV mask - debug - add or change comments --- .../lightingEstimation/ellipseGeometry.cpp | 109 ++++++++++-------- 1 file changed, 60 insertions(+), 49 deletions(-) diff --git a/src/aliceVision/lightingEstimation/ellipseGeometry.cpp b/src/aliceVision/lightingEstimation/ellipseGeometry.cpp index 7528027d2d..31494df32d 100644 --- a/src/aliceVision/lightingEstimation/ellipseGeometry.cpp +++ b/src/aliceVision/lightingEstimation/ellipseGeometry.cpp @@ -126,80 +126,66 @@ void sphereRayIntersection(const Eigen::Vector3f& direction, void estimateSphereNormals(const std::array& sphereCenter, const float sphereRadius, - const image::Image& ellipseMask, const Eigen::Matrix3f& K, image::Image& normals, image::Image& newMask) { Eigen::Matrix3f invK = K.inverse(); - bool print = false; - - for (int i = 1; i < ellipseMask.rows(); ++i) { - for (int j = 1; j < ellipseMask.cols(); ++j) { - if (ellipseMask(i, j) == 1) { - // Get homogeneous coordinates of the pixel : - Eigen::Vector3f coordinates = Eigen::Vector3f(j, i, 1.0f); - - // Get the direction of the ray : - Eigen::Vector3f direction = invK*coordinates; - - // Estimate the interception of the ray with the sphere : - float delta = 0.0; - Eigen::Vector3f normal = Eigen::Vector3f(0.0f, 0.0f, 0.0f); - - print = false; - - sphereRayIntersection(direction, sphereCenter, sphereRadius, delta, normal); - - // If the ray intercepts the sphere we add this pixel to the new mask : - if(delta > 0) - { - newMask(i, j) = 1.0; - normals(i, j) = image::RGBfColor(normal[0], normal[1], normal[2]); - } - else - { - newMask(i, j) = 0.0; - normals(i, j) = image::RGBfColor(0.0, 0.0, 0.0); - } + + for (int i = 0; i < normals.rows(); ++i) { + for (int j = 0; j < normals.cols(); ++j) { + // Get homogeneous coordinates of the pixel : + Eigen::Vector3f coordinates = Eigen::Vector3f(j, i, 1.0f); + + // Get the direction of the ray : + Eigen::Vector3f direction = invK*coordinates; + + // Estimate the interception of the ray with the sphere : + float delta = 0.0; + Eigen::Vector3f normal = Eigen::Vector3f(0.0f, 0.0f, 0.0f); + + sphereRayIntersection(direction, sphereCenter, sphereRadius, delta, normal); + + // If the ray intercepts the sphere we add this pixel to the new mask : + if(delta > 0) + { + newMask(i, j) = 1.0; + normals(i, j) = image::RGBfColor(normal[0], normal[1], normal[2]); + } + else + { + newMask(i, j) = 0.0; + normals(i, j) = image::RGBfColor(0.0, 0.0, 0.0); } } } } -void getRealNormalOnSphere(const std::string& maskPath, +void getRealNormalOnSphere(const cv::Mat& maskCV, const Eigen::Matrix3f& K, const float sphereRadius, image::Image& normals, image::Image& newMask) { - // Charger l'image depuis le chemin spécifié - cv::Mat img = cv::imread(maskPath); - // Convertir l'image en niveaux de gris - cv::Mat gray; - cv::cvtColor(img, gray, cv::COLOR_BGR2GRAY); - - // Appliquer un seuillage pour convertir l'image en noir et blanc + // Apply a threshold to the image int thresholdValue = 150; cv::Mat thresh; - cv::threshold(gray, thresh, thresholdValue, 255, 0); + cv::threshold(maskCV, thresh, thresholdValue, 255, 0); - // Trouver les contours + // Find contours std::vector> contours; std::vector hierarchy; cv::findContours(thresh, contours, hierarchy, cv::RETR_TREE, cv::CHAIN_APPROX_SIMPLE); - // Sélectionner le premier contour + // Fit the ellipse to the first contour std::vector cnt = contours[0]; - - // Ajuster l'ellipse sur l'objet sélectionné cv::RotatedRect ellipse = cv::fitEllipse(cnt); float angle = ellipse.angle; cv::Size2f size = ellipse.size; cv::Point2f center = ellipse.center; - // Convertir l'ellipse en un tableau de 5 paramètres: + // Ellipse is converted as five-parameter array std::array ellipseParameters; ellipseParameters[0] = angle; @@ -210,10 +196,35 @@ void getRealNormalOnSphere(const std::string& maskPath, std::array sphereCenter; estimateSphereCenter(ellipseParameters, sphereRadius, K, sphereCenter); - - image::Image mask; - photometricStereo::loadMask(maskPath, mask); - estimateSphereNormals(sphereCenter, sphereRadius, mask, K, normals, newMask); + estimateSphereNormals(sphereCenter, sphereRadius, K, normals, newMask); + +void getEllipseMaskFromSphereParameters(const std::array& sphereParam, const Eigen::Matrix3f& K, std::array& ellipseParameters, cv::Mat maskCV) +{ + + // Distance between center of image and center of disk : + float imX = K(0, 2); + float imY = K(1, 2); + float f = K(0, 0); + float delta = sqrt((sphereParam[0])*(sphereParam[0]) + (sphereParam[1])*(sphereParam[1])); + // sphere params = x,y, radius + // ellipse params = angle, x, y, semi minor axe, semi major axe + + // Ellipse from sphere parameters + // semi minor axe = radius; + // semi major axe = formula from paper + // main direction = disc center to picture center + // ellipse center = disc center + + float radians = atan((sphereParam[0]) / (sphereParam[1])); + ellipseParameters[0] = radians * (180.0/3.141592653589793238463); + + ellipseParameters[1] = sphereParam[0]; + ellipseParameters[2] = sphereParam[1]; + ellipseParameters[3] = sphereParam[2]; + + + // a² = b² ((distance between image center and disc center)² + f² + b²)/(f² + b²) + ellipseParameters[4] = sqrt((ellipseParameters[3]*ellipseParameters[3]) * (delta * delta + f*f + ellipseParameters[3]*ellipseParameters[3])/(f*f + ellipseParameters[3]*ellipseParameters[3])); } } // namespace lightingEstimation From 8de36df6b87ba62bb82da66ddc037a334e4388a5 Mon Sep 17 00:00:00 2001 From: jmelou Date: Mon, 29 Apr 2024 11:55:29 +0200 Subject: [PATCH 44/67] [lightingCaliration] Forgotten in the previous commit --- src/aliceVision/lightingEstimation/ellipseGeometry.cpp | 1 + src/aliceVision/lightingEstimation/ellipseGeometry.hpp | 10 +++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/aliceVision/lightingEstimation/ellipseGeometry.cpp b/src/aliceVision/lightingEstimation/ellipseGeometry.cpp index 31494df32d..20a2bbcc7d 100644 --- a/src/aliceVision/lightingEstimation/ellipseGeometry.cpp +++ b/src/aliceVision/lightingEstimation/ellipseGeometry.cpp @@ -197,6 +197,7 @@ void getRealNormalOnSphere(const cv::Mat& maskCV, std::array sphereCenter; estimateSphereCenter(ellipseParameters, sphereRadius, K, sphereCenter); estimateSphereNormals(sphereCenter, sphereRadius, K, normals, newMask); +} void getEllipseMaskFromSphereParameters(const std::array& sphereParam, const Eigen::Matrix3f& K, std::array& ellipseParameters, cv::Mat maskCV) { diff --git a/src/aliceVision/lightingEstimation/ellipseGeometry.hpp b/src/aliceVision/lightingEstimation/ellipseGeometry.hpp index 34c9c0a08c..954e293ea3 100644 --- a/src/aliceVision/lightingEstimation/ellipseGeometry.hpp +++ b/src/aliceVision/lightingEstimation/ellipseGeometry.hpp @@ -8,6 +8,9 @@ #include +#include +#include + #include #include #include @@ -48,24 +51,25 @@ int findUniqueIndex(const std::vector& vec); */ void estimateSphereNormals(const std::array& sphereCenter, const float sphereRadius, - const image::Image& ellipseMask, const Eigen::Matrix3f& K, image::Image& normals, image::Image& newMask); /** * @brief Estimate the normals of a sphere from a mask - * @param[in] maskPath The path to the binary mask of the ellipse in the picture + * @param[in] maskCV The openCV image of the binary mask of the ellipse in the picture * @param[in] K Intrinsic parameters of the camera * @param[out] normals Normals on the sphere in camera frame * @param[out] newMask The mask of the sphere after ray tracing */ -void getRealNormalOnSphere(const std::string& maskPath, +void getRealNormalOnSphere(const cv::Mat& maskCV, const Eigen::Matrix3f& K, const float sphereRadius, image::Image& normals, image::Image& newMask); +void getEllipseMaskFromSphereParameters(const std::array& sphereParam, const Eigen::Matrix3f& K, std::array& ellipseParameters, cv::Mat maskCV); + } // namespace lightingEstimation } // namespace aliceVision \ No newline at end of file From 54f1f6422247472d4ad67323ebf5f6e21f89a3c9 Mon Sep 17 00:00:00 2001 From: jmelou Date: Mon, 29 Apr 2024 11:55:55 +0200 Subject: [PATCH 45/67] [lightingCalibration] Add ellipseGeometry in cmakelist --- src/aliceVision/lightingEstimation/CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/aliceVision/lightingEstimation/CMakeLists.txt b/src/aliceVision/lightingEstimation/CMakeLists.txt index 34297a7a07..011d7ef881 100644 --- a/src/aliceVision/lightingEstimation/CMakeLists.txt +++ b/src/aliceVision/lightingEstimation/CMakeLists.txt @@ -3,6 +3,7 @@ set(lightingEstimation_files_headers augmentedNormals.hpp lightingEstimation.hpp lightingCalibration.hpp + ellipseGeometry.hpp ) # Sources @@ -10,13 +11,16 @@ set(lightingEstimation_files_sources augmentedNormals.cpp lightingEstimation.cpp lightingCalibration.cpp + ellipseGeometry.cpp ) alicevision_add_library(aliceVision_lightingEstimation SOURCES ${lightingEstimation_files_headers} ${lightingEstimation_files_sources} PUBLIC_LINKS + ${OpenCV_LIBS} aliceVision_image aliceVision_system + aliceVision_photometricStereo ) From 1cdf820d676195b60bfa370ea39f51cd50a79612 Mon Sep 17 00:00:00 2001 From: jmelou Date: Thu, 2 May 2024 13:46:53 +0200 Subject: [PATCH 46/67] [NormalIntegration] Debug normal integration --- src/aliceVision/photometricStereo/normalIntegration.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aliceVision/photometricStereo/normalIntegration.cpp b/src/aliceVision/photometricStereo/normalIntegration.cpp index b6ae41250a..f14f01e7b3 100644 --- a/src/aliceVision/photometricStereo/normalIntegration.cpp +++ b/src/aliceVision/photometricStereo/normalIntegration.cpp @@ -348,7 +348,7 @@ void DCTIntegration(const image::Image& normals, { double denom = 4 * (pow(sin(0.5 * M_PI * j / nbCols), 2) + pow(sin(0.5 * M_PI * i / nbRows), 2)); denom = std::max(denom, 0.0001); - z_bar_bar.at(i, j) = fcos.at(i, j) / denom; + z_bar_bar.at(i, j) = -fcos.at(i, j) / denom; } } From 173acbbb00b0b7efb04caccdd0b5c4f0071e253d Mon Sep 17 00:00:00 2001 From: jmelou Date: Fri, 31 May 2024 11:32:05 +0200 Subject: [PATCH 47/67] [PS - I/O] Add new function for mask reading This new function creates an image which contains, for each pixel, its position in the mask --- .../photometricStereo/photometricDataIO.cpp | 23 +++++++++++++++++++ .../photometricStereo/photometricDataIO.hpp | 2 ++ 2 files changed, 25 insertions(+) diff --git a/src/aliceVision/photometricStereo/photometricDataIO.cpp b/src/aliceVision/photometricStereo/photometricDataIO.cpp index 80ee00e52b..eb75a7aafb 100644 --- a/src/aliceVision/photometricStereo/photometricDataIO.cpp +++ b/src/aliceVision/photometricStereo/photometricDataIO.cpp @@ -354,6 +354,29 @@ void getIndMask(image::Image const& mask, std::vector& indices) } } +void getIndMask(image::Image const& mask, std::vector& indices, image::Image& indexInMask) +{ + const int nbRows = mask.rows(); + const int nbCols = mask.cols(); + + for (int j = 0; j < nbCols; ++j) + { + for (int i = 0; i < nbRows; ++i) + { + if (mask(i, j) > 0.7) + { + int currentIndex = j * nbRows + i; + indices.push_back(currentIndex); + indexInMask(i, j) = indices.size() - 1; + } + else + { + indexInMask(i, j) = -1; + } + } + } +} + void intensityScaling(std::array const& intensities, image::Image& imageToScale) { int nbRows = imageToScale.rows(); diff --git a/src/aliceVision/photometricStereo/photometricDataIO.hpp b/src/aliceVision/photometricStereo/photometricDataIO.hpp index 7bb4908ed4..18d9e39467 100644 --- a/src/aliceVision/photometricStereo/photometricDataIO.hpp +++ b/src/aliceVision/photometricStereo/photometricDataIO.hpp @@ -85,6 +85,8 @@ void loadMask(std::string const& maskName, image::Image& mask); */ void getIndMask(image::Image const& mask, std::vector& indices); +void getIndMask(image::Image const& mask, std::vector& indices, image::Image& indexInMask); + /** * @brief Apply the intensities to each channel of each image * @param[in] intensities Intensity values to apply to the image From b74da78e043c6246c130cddf899caf93f4cb119e Mon Sep 17 00:00:00 2001 From: jmelou Date: Fri, 31 May 2024 11:39:50 +0200 Subject: [PATCH 48/67] [PS] Debug read of a built model --- .../photometricStereo/photometricStereo.cpp | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/aliceVision/photometricStereo/photometricStereo.cpp b/src/aliceVision/photometricStereo/photometricStereo.cpp index ce1510a2bd..143f2522bf 100644 --- a/src/aliceVision/photometricStereo/photometricStereo.cpp +++ b/src/aliceVision/photometricStereo/photometricStereo.cpp @@ -118,9 +118,10 @@ void photometricStereo(const sfmData::SfMData& sfmData, hasMetadata = false; std::vector viewIds; + std::map idMap; + if (hasMetadata) { - std::map idMap; for (auto& viewId : initViewIds) { std::map currentMetadata = sfmData.getView(viewId).getImage().getMetadata(); @@ -134,7 +135,15 @@ void photometricStereo(const sfmData::SfMData& sfmData, } else { - viewIds = initViewIds; + for (auto& viewId : initViewIds) + { + const fs::path imagePath = fs::path(sfmData.getView(viewId).getImage().getImagePath()); + idMap[imagePath] = viewId; + } + } + for (const auto& [currentId, viewId] : idMap) + { + viewIds.push_back(viewId); } for (auto& viewId : viewIds) From 070420d269b27b03ed0186dc4805f251c6d831c7 Mon Sep 17 00:00:00 2001 From: jmelou Date: Fri, 31 May 2024 11:40:32 +0200 Subject: [PATCH 49/67] [PS] Debug sub-masks creation --- src/aliceVision/photometricStereo/photometricStereo.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/aliceVision/photometricStereo/photometricStereo.cpp b/src/aliceVision/photometricStereo/photometricStereo.cpp index 143f2522bf..a57328bc66 100644 --- a/src/aliceVision/photometricStereo/photometricStereo.cpp +++ b/src/aliceVision/photometricStereo/photometricStereo.cpp @@ -363,11 +363,12 @@ void photometricStereo(const std::vector& imageList, int numberOfMasks = 1; - while (numberOfPixels > sizeMax / imageList.size()) + while (numberOfPixels > (sizeMax / imageList.size())) { numberOfMasks = numberOfMasks * 2; auxMaskSize = floor(auxMaskSize / 2); - numberOfPixels = 3 * auxMaskSize; + numberOfPixels = floor(numberOfPixels / 2); + hasMask = true; } Eigen::MatrixXf normalsVect = Eigen::MatrixXf::Zero(3, pictRows * pictCols); From 4aff764efca55756a1bfd6e681381fc304839738 Mon Sep 17 00:00:00 2001 From: jmelou Date: Sun, 2 Jun 2024 23:16:38 +0200 Subject: [PATCH 50/67] [PS] Correct spelling of "ambient" --- .../lightingCalibration.cpp | 8 +++---- .../lightingEstimation/lightingEstimation.cpp | 2 +- .../photometricStereo/photometricDataIO.cpp | 6 ++--- .../photometricStereo/photometricStereo.cpp | 22 +++++++++---------- .../photometricStereo/photometricStereo.hpp | 6 ++--- .../sphereDetection/sphereDetection.cpp | 2 +- src/software/pipeline/main_cameraInit.cpp | 12 +++++----- .../pipeline/main_photometricStereo.cpp | 6 ++--- 8 files changed, 32 insertions(+), 32 deletions(-) diff --git a/src/aliceVision/lightingEstimation/lightingCalibration.cpp b/src/aliceVision/lightingEstimation/lightingCalibration.cpp index 947e8b4089..823a1f2f21 100644 --- a/src/aliceVision/lightingEstimation/lightingCalibration.cpp +++ b/src/aliceVision/lightingEstimation/lightingCalibration.cpp @@ -87,7 +87,7 @@ void lightCalibration(const sfmData::SfMData& sfmData, ALICEVISION_LOG_INFO("View Id: " << currentView.getViewId()); const fs::path imagePath = fs::path(currentView.getImage().getImagePath()); - if (!boost::algorithm::icontains(imagePath.stem().string(), "ambiant")) + if (!boost::algorithm::icontains(imagePath.stem().string(), "ambient")) { std::string sphereName = std::to_string(currentView.getViewId()); auto sphereExists = (fileTree.get_child_optional(sphereName)).is_initialized(); @@ -138,7 +138,7 @@ void lightCalibration(const sfmData::SfMData& sfmData, ALICEVISION_LOG_INFO("View Id: " << currentView.getViewId()); const fs::path imagePath = fs::path(currentView.getImage().getImagePath()); - if (!boost::algorithm::icontains(imagePath.stem().string(), "ambiant")) + if (!boost::algorithm::icontains(imagePath.stem().string(), "ambient")) { ALICEVISION_LOG_INFO(" - " << imagePath.string()); imageList.push_back(imagePath.string()); @@ -641,7 +641,7 @@ void writeJSON(const std::string& fileName, const bool calibratedFile = (std::find(imageList.begin(), imageList.end(), viewId.getImage().getImagePath()) != imageList.end()); // Only write images that were actually used for the lighting calibration, instead of all the input images - if (!boost::algorithm::icontains(imagePath.stem().string(), "ambiant") && calibratedFile) + if (!boost::algorithm::icontains(imagePath.stem().string(), "ambient") && calibratedFile) { bpt::ptree lightTree; if (saveAsModel) @@ -684,7 +684,7 @@ void writeJSON(const std::string& fileName, else { ALICEVISION_LOG_INFO("'" << imagePath << "' is in the input SfMData but has not been used for the lighting " - << "calibration or contains 'ambiant' in its filename."); + << "calibration or contains 'ambient' in its filename."); } } diff --git a/src/aliceVision/lightingEstimation/lightingEstimation.cpp b/src/aliceVision/lightingEstimation/lightingEstimation.cpp index e0886bceec..359b9b38e1 100644 --- a/src/aliceVision/lightingEstimation/lightingEstimation.cpp +++ b/src/aliceVision/lightingEstimation/lightingEstimation.cpp @@ -32,7 +32,7 @@ void albedoNormalsProduct(MatrixXf& rhoTimesN, const MatrixXf& albedoChannel, co rhoTimesN(validIndex, 0) = albedoChannel(i) * augmentedNormals(i).nx(); rhoTimesN(validIndex, 1) = albedoChannel(i) * augmentedNormals(i).ny(); rhoTimesN(validIndex, 2) = albedoChannel(i) * augmentedNormals(i).nz(); - rhoTimesN(validIndex, 3) = albedoChannel(i) * augmentedNormals(i).nambiant(); + rhoTimesN(validIndex, 3) = albedoChannel(i) * augmentedNormals(i).nambient(); rhoTimesN(validIndex, 4) = albedoChannel(i) * augmentedNormals(i).nx_ny(); rhoTimesN(validIndex, 5) = albedoChannel(i) * augmentedNormals(i).nx_nz(); rhoTimesN(validIndex, 6) = albedoChannel(i) * augmentedNormals(i).ny_nz(); diff --git a/src/aliceVision/photometricStereo/photometricDataIO.cpp b/src/aliceVision/photometricStereo/photometricDataIO.cpp index eb75a7aafb..77c0dfe400 100644 --- a/src/aliceVision/photometricStereo/photometricDataIO.cpp +++ b/src/aliceVision/photometricStereo/photometricDataIO.cpp @@ -98,7 +98,7 @@ void loadLightSH(const std::string& dirFileName, Eigen::MatrixXf& lightMat) { std::stringstream stream; std::string line; - float x, y, z, ambiant, nxny, nxnz, nynz, nx2ny2, nz2; + float x, y, z, ambient, nxny, nxnz, nynz, nx2ny2, nz2; std::fstream dirFile; dirFile.open(dirFileName, std::ios::in); @@ -118,13 +118,13 @@ void loadLightSH(const std::string& dirFileName, Eigen::MatrixXf& lightMat) stream.clear(); stream.str(line); - stream >> x >> y >> z >> ambiant >> nxny >> nxnz >> nynz >> nx2ny2 >> nz2; + stream >> x >> y >> z >> ambient >> nxny >> nxnz >> nynz >> nx2ny2 >> nz2; if (lineNumber < lightMat.rows()) { lightMat(lineNumber, 0) = x; lightMat(lineNumber, 1) = -y; lightMat(lineNumber, 2) = -z; - lightMat(lineNumber, 3) = ambiant; + lightMat(lineNumber, 3) = ambient; lightMat(lineNumber, 4) = nxny; lightMat(lineNumber, 5) = nxnz; lightMat(lineNumber, 6) = nynz; diff --git a/src/aliceVision/photometricStereo/photometricStereo.cpp b/src/aliceVision/photometricStereo/photometricStereo.cpp index a57328bc66..07a7e3e49c 100644 --- a/src/aliceVision/photometricStereo/photometricStereo.cpp +++ b/src/aliceVision/photometricStereo/photometricStereo.cpp @@ -61,21 +61,21 @@ void photometricStereo(const std::string& inputPath, std::string maskName = lightDataPath.remove_filename().string() + "/mask.png"; loadMask(maskName, mask); - std::string pathToAmbiant = ""; + std::string pathToAmbient = ""; - if (PSParameters.removeAmbiant) + if (PSParameters.removeAmbient) { for (auto& currentPath : imageList) { const fs::path imagePath = fs::path(currentPath); - if (boost::algorithm::icontains(imagePath.stem().string(), "ambiant")) + if (boost::algorithm::icontains(imagePath.stem().string(), "ambient")) { - pathToAmbiant = imagePath.string(); + pathToAmbient = imagePath.string(); } } } - photometricStereo(imageList, intList, lightMat, mask, pathToAmbiant, PSParameters, normals, albedo); + photometricStereo(imageList, intList, lightMat, mask, pathToAmbient, PSParameters, normals, albedo); writePSResults(outputPath, normals, albedo); image::writeImage(outputPath + "/mask.png", mask, image::ImageWriteOptions().toColorSpace(image::EImageColorSpace::NO_CONVERSION)); @@ -97,7 +97,7 @@ void photometricStereo(const sfmData::SfMData& sfmData, ALICEVISION_LOG_INFO("SH will soon be available for use in PS. For now, lighting is reduced to directional"); } - std::string pathToAmbiant = ""; + std::string pathToAmbient = ""; std::map> viewsPerPoseId; for (auto& viewIt : sfmData.getViews()) @@ -149,15 +149,15 @@ void photometricStereo(const sfmData::SfMData& sfmData, for (auto& viewId : viewIds) { const fs::path imagePath = fs::path(sfmData.getView(viewId).getImage().getImagePath()); - if (!boost::algorithm::icontains(imagePath.stem().string(), "ambiant")) + if (!boost::algorithm::icontains(imagePath.stem().string(), "ambient")) { ALICEVISION_LOG_INFO(" - " << imagePath.string()); imageList.push_back(imagePath.string()); } - else if (PSParameters.removeAmbiant) + else if (PSParameters.removeAmbient) { - ALICEVISION_LOG_INFO("Remove ambiant light - " << imagePath.string()); - pathToAmbiant = imagePath.string(); + ALICEVISION_LOG_INFO("Remove ambient light - " << imagePath.string()); + pathToAmbient = imagePath.string(); } } @@ -211,7 +211,7 @@ void photometricStereo(const sfmData::SfMData& sfmData, loadMask(currentMaskPath, mask); - photometricStereo(imageList, intList, lightMat, mask, pathToAmbiant, PSParameters, normals, albedo); + photometricStereo(imageList, intList, lightMat, mask, pathToAmbient, PSParameters, normals, albedo); writePSResults(outputPath, normals, albedo, posesIt.first); image::writeImage(outputPath + "/" + std::to_string(posesIt.first) + "_mask.png", diff --git a/src/aliceVision/photometricStereo/photometricStereo.hpp b/src/aliceVision/photometricStereo/photometricStereo.hpp index cd5a1eda86..3b72721cc1 100644 --- a/src/aliceVision/photometricStereo/photometricStereo.hpp +++ b/src/aliceVision/photometricStereo/photometricStereo.hpp @@ -19,7 +19,7 @@ namespace photometricStereo { struct PhotometricSteroParameters { size_t SHOrder; // Order of spherical harmonics (lighting) - bool removeAmbiant; // Do we remove ambiant light ? (currently tested) + bool removeAmbient; // Do we remove ambient light ? (currently tested) bool isRobust; // Do we use the robust version of the algorithm ? (currently tested) int downscale; // Downscale factor }; @@ -65,7 +65,7 @@ void photometricStereo(const sfmData::SfMData& sfmData, * @param[in] intList List of light intensities * @param[in] lightMat List of light direction/coefficients (SH) * @param[in] mask Mask that defines region of interest - * @param[in] pathToAmbiant Path to picture without any additional lighting + * @param[in] pathToAmbient Path to picture without any additional lighting * @param[in] PSParameters Parameters for the PS algorithm * @param[out] normals Normal map of the scene * @param[out] albedo Albedo map of the scene @@ -74,7 +74,7 @@ void photometricStereo(const std::vector& imageList, const std::vector>& intList, const Eigen::MatrixXf& lightMat, image::Image& mask, - const std::string& pathToAmbiant, + const std::string& pathToAmbient, const PhotometricSteroParameters& PSParameters, image::Image& normals, image::Image& albedo); diff --git a/src/aliceVision/sphereDetection/sphereDetection.cpp b/src/aliceVision/sphereDetection/sphereDetection.cpp index fb0c9e5d6b..83683ca3ff 100644 --- a/src/aliceVision/sphereDetection/sphereDetection.cpp +++ b/src/aliceVision/sphereDetection/sphereDetection.cpp @@ -183,7 +183,7 @@ void sphereDetection(const sfmData::SfMData& sfmData, Ort::Session& session, fs: const std::string sphereName = std::to_string(viewID.second->getViewId()); const fs::path imagePath = fs::path(sfmData.getView(viewID.second->getViewId()).getImage().getImagePath()); - if (boost::algorithm::icontains(imagePath.stem().string(), "ambiant")) + if (boost::algorithm::icontains(imagePath.stem().string(), "ambient")) continue; const auto pred = predict(session, imagePath, minScore); diff --git a/src/software/pipeline/main_cameraInit.cpp b/src/software/pipeline/main_cameraInit.cpp index 980dbd853a..d2c76092b1 100644 --- a/src/software/pipeline/main_cameraInit.cpp +++ b/src/software/pipeline/main_cameraInit.cpp @@ -805,24 +805,24 @@ int aliceVision_main(int argc, char** argv) { for (const auto& poseGroup : poseGroups) { - bool hasAmbiant = false; + bool hasAmbient = false; - // Photometric stereo : ambiant viewId used for all pictures + // Photometric stereo : ambient viewId used for all pictures for (const IndexT vId : poseGroup.second) { const fs::path imagePath = fs::path(sfmData.getView(vId).getImage().getImagePath()); - if (boost::algorithm::icontains(imagePath.stem().string(), "ambiant")) + if (boost::algorithm::icontains(imagePath.stem().string(), "ambient")) { - hasAmbiant = true; + hasAmbient = true; for (const auto it : poseGroup.second) { - // Update poseId with ambiant view id + // Update poseId with ambient view id sfmData.getView(it).setPoseId(vId); } break; } } - if (!hasAmbiant) + if (!hasAmbient) { // Sort views of the poseGroup per timestamps std::vector> sortedViews; diff --git a/src/software/pipeline/main_photometricStereo.cpp b/src/software/pipeline/main_photometricStereo.cpp index 674671912b..4b605bb840 100644 --- a/src/software/pipeline/main_photometricStereo.cpp +++ b/src/software/pipeline/main_photometricStereo.cpp @@ -78,9 +78,9 @@ int aliceVision_main(int argc, char** argv) ("pathToJSONLightFile,l", po::value(&pathToLightData)->default_value("defaultJSON.txt"), "Path to light file (JSON). If empty, .txt files are expected in the image folder.") ("SHOrder,s", po::value(&PSParameters.SHOrder)->default_value(0), - "Spherical harmonics order, 0 = directional, 1 = directional + ambiant, 2 = second order SH.") - ("removeAmbiant,a", po::value(&PSParameters.removeAmbiant)->default_value(false), - "True if the ambiant light is to be removed on PS images, false otherwise.") + "Spherical harmonics order, 0 = directional, 1 = directional + ambient, 2 = second order SH.") + ("removeAmbient,a", po::value(&PSParameters.removeAmbient)->default_value(false), + "True if the ambient light is to be removed on PS images, false otherwise.") ("isRobust,r", po::value(&PSParameters.isRobust)->default_value(false), "True to use the robust algorithm, false otherwise.") ("downscale, d", po::value(&PSParameters.downscale)->default_value(1), From e0177efaba659549993c2d00e7fbe3a6a8d33968 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Tue, 4 Jun 2024 00:38:25 +0200 Subject: [PATCH 51/67] build fixes --- src/aliceVision/lightingEstimation/augmentedNormals.hpp | 4 ++-- src/aliceVision/lightingEstimation/lightingCalibration.cpp | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/aliceVision/lightingEstimation/augmentedNormals.hpp b/src/aliceVision/lightingEstimation/augmentedNormals.hpp index e98bba686e..4eb7de4bba 100644 --- a/src/aliceVision/lightingEstimation/augmentedNormals.hpp +++ b/src/aliceVision/lightingEstimation/augmentedNormals.hpp @@ -67,8 +67,8 @@ class AugmentedNormal : public Eigen::Matrix inline const T& nz() const { return (*this)(2); } inline T& nz() { return (*this)(2); } - inline const T& nambiant() const { return (*this)(3); } - inline T& nambiant() { return (*this)(3); } + inline const T& nambient() const { return (*this)(3); } + inline T& nambient() { return (*this)(3); } inline const T& nx_ny() const { return (*this)(4); } inline T& nx_ny() { return (*this)(4); } diff --git a/src/aliceVision/lightingEstimation/lightingCalibration.cpp b/src/aliceVision/lightingEstimation/lightingCalibration.cpp index 823a1f2f21..0384bc7fd6 100644 --- a/src/aliceVision/lightingEstimation/lightingCalibration.cpp +++ b/src/aliceVision/lightingEstimation/lightingCalibration.cpp @@ -386,6 +386,8 @@ void lightCalibrationOneImage(const std::string& picturePath, lightingDirection << directionnalPart, secondOrder; } +} + void calibrateLightFromRealSphere(const image::Image& imageFloat, const cv::Mat& maskCV, const Eigen::Matrix3f& K, From 0a9485c480ff1b4fa2690652239bba424ba0fb5d Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Tue, 4 Jun 2024 01:01:54 +0200 Subject: [PATCH 52/67] [bin] lidarMeshing: build fix related to MultiViewParams constructor changes --- src/software/pipeline/main_lidarMeshing.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/software/pipeline/main_lidarMeshing.cpp b/src/software/pipeline/main_lidarMeshing.cpp index 1d717ec021..bb2d94a25f 100644 --- a/src/software/pipeline/main_lidarMeshing.cpp +++ b/src/software/pipeline/main_lidarMeshing.cpp @@ -49,7 +49,7 @@ bool computeSubMesh(const std::string& pathSfmData, std::string& outputFile, con ALICEVISION_LOG_INFO("Loading source done"); // Create multiview params - mvsUtils::MultiViewParams mp(sfmData, "", "", "", false); + mvsUtils::MultiViewParams mp(sfmData); mp.userParams.put("LargeScale.forcePixelSize", 0.01); mp.userParams.put("LargeScale.forceWeight", 32.0); mp.userParams.put("LargeScale.helperPointsGridSize", 10); From 1aec3b0788e4d70786dec75401f087cbf8356f15 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Tue, 4 Jun 2024 18:38:10 +0200 Subject: [PATCH 53/67] [cmake] include order --- src/aliceVision/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aliceVision/CMakeLists.txt b/src/aliceVision/CMakeLists.txt index fbbdfaf3e0..25d82941d8 100644 --- a/src/aliceVision/CMakeLists.txt +++ b/src/aliceVision/CMakeLists.txt @@ -51,7 +51,6 @@ endif() # MVS modules if(ALICEVISION_BUILD_MVS) - add_subdirectory(lightingEstimation) add_subdirectory(mesh) add_subdirectory(mvsData) add_subdirectory(mvsUtils) @@ -79,6 +78,7 @@ if (ALICEVISION_BUILD_STEREOPHOTOMETRY) add_subdirectory(sphereDetection) endif() endif() + add_subdirectory(lightingEstimation) endif() From 3263e1f21982bdacbfb6645104e72b9855417d18 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Wed, 5 Jun 2024 10:41:07 +0200 Subject: [PATCH 54/67] [cmake] minor build cleanup --- src/CMakeLists.txt | 4 ++-- src/aliceVision/CMakeLists.txt | 4 ++-- src/software/pipeline/CMakeLists.txt | 25 +++++++++++++------------ 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c8e5ea5861..863c5a7d76 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -51,7 +51,7 @@ option(ALICEVISION_BUILD_SFM "Build AliceVision SfM part" ON) option(ALICEVISION_BUILD_MVS "Build AliceVision MVS part" ON) option(ALICEVISION_BUILD_HDR "Build AliceVision HDR part" ON) option(ALICEVISION_BUILD_SEGMENTATION "Build AliceVision Segmentation part" ON) -option(ALICEVISION_BUILD_STEREOPHOTOMETRY "Build AliceVision StereoPhotometry part" ON) +option(ALICEVISION_BUILD_PHOTOMETRICSTEREO "Build AliceVision Photometric Stereo part" ON) option(ALICEVISION_BUILD_PANORAMA "Build AliceVision panorama part" ON) option(ALICEVISION_BUILD_SOFTWARE "Build AliceVision command line tools." ON) option(ALICEVISION_BUILD_COVERAGE "Enable code coverage generation (gcc only)" OFF) @@ -94,7 +94,7 @@ if(NOT ALICEVISION_BUILD_SFM) SET(ALICEVISION_BUILD_MVS OFF) SET(ALICEVISION_BUILD_HDR OFF) SET(ALICEVISION_BUILD_SEGMENTATION OFF) - SET(ALICEVISION_BUILD_STEREOPHOTOMETRY OFF) + SET(ALICEVISION_BUILD_PHOTOMETRICSTEREO OFF) SET(ALICEVISION_BUILD_PANORAMA OFF) SET(ALICEVISION_BUILD_LIDAR OFF) endif() diff --git a/src/aliceVision/CMakeLists.txt b/src/aliceVision/CMakeLists.txt index 25d82941d8..9e5dd9a62c 100644 --- a/src/aliceVision/CMakeLists.txt +++ b/src/aliceVision/CMakeLists.txt @@ -71,14 +71,14 @@ if(ALICEVISION_BUILD_SFM AND ALICEVISION_BUILD_MVS) add_subdirectory(sfmMvsUtils) endif() -if (ALICEVISION_BUILD_STEREOPHOTOMETRY) +if (ALICEVISION_BUILD_PHOTOMETRICSTEREO) if(ALICEVISION_HAVE_OPENCV) add_subdirectory(photometricStereo) if(ALICEVISION_HAVE_ONNX) add_subdirectory(sphereDetection) endif() + add_subdirectory(lightingEstimation) endif() - add_subdirectory(lightingEstimation) endif() diff --git a/src/software/pipeline/CMakeLists.txt b/src/software/pipeline/CMakeLists.txt index ea06fe9944..b405bd1942 100644 --- a/src/software/pipeline/CMakeLists.txt +++ b/src/software/pipeline/CMakeLists.txt @@ -625,18 +625,7 @@ if(ALICEVISION_BUILD_MVS) endif() # if(ALICEVISION_BUILD_MVS) -if (ALICEVISION_BUILD_STEREOPHOTOMETRY) - alicevision_add_software(aliceVision_lightingCalibration - SOURCE main_lightingCalibration.cpp - FOLDER ${FOLDER_SOFTWARE_PIPELINE} - LINKS aliceVision_lightingEstimation - aliceVision_cmdline - aliceVision_system - aliceVision_sfmData - aliceVision_sfmDataIO - Boost::program_options - ) - +if (ALICEVISION_BUILD_PHOTOMETRICSTEREO) if(ALICEVISION_HAVE_OPENCV) # Photometric Stereo @@ -682,6 +671,18 @@ if (ALICEVISION_BUILD_STEREOPHOTOMETRY) ${OpenCV_LIBRARIES} ) endif() + + alicevision_add_software(aliceVision_lightingCalibration + SOURCE main_lightingCalibration.cpp + FOLDER ${FOLDER_SOFTWARE_PIPELINE} + LINKS aliceVision_lightingEstimation + aliceVision_cmdline + aliceVision_system + aliceVision_sfmData + aliceVision_sfmDataIO + Boost::program_options + ) + endif() endif() From 9b99573312206e657b1d322bfbb956b8d0112768 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Wed, 5 Jun 2024 12:29:46 +0200 Subject: [PATCH 55/67] [photometricStereo] msvc build fix --- src/aliceVision/photometricStereo/photometricStereo.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/aliceVision/photometricStereo/photometricStereo.cpp b/src/aliceVision/photometricStereo/photometricStereo.cpp index 07a7e3e49c..dbd6889e0d 100644 --- a/src/aliceVision/photometricStereo/photometricStereo.cpp +++ b/src/aliceVision/photometricStereo/photometricStereo.cpp @@ -138,7 +138,7 @@ void photometricStereo(const sfmData::SfMData& sfmData, for (auto& viewId : initViewIds) { const fs::path imagePath = fs::path(sfmData.getView(viewId).getImage().getImagePath()); - idMap[imagePath] = viewId; + idMap[imagePath.string()] = viewId; } } for (const auto& [currentId, viewId] : idMap) @@ -170,7 +170,7 @@ void photometricStereo(const sfmData::SfMData& sfmData, } else { - const std::string extension = fs::path(lightData).extension(); + const std::string extension = fs::path(lightData).extension().string(); if (extension == ".json") // JSON File { From 7dc46c3505999646554a84142180908691390398 Mon Sep 17 00:00:00 2001 From: jmelou Date: Wed, 5 Jun 2024 15:39:36 +0200 Subject: [PATCH 56/67] [lightingCalibration] Add option for elliptic estimation --- .../lightingEstimation/lightingCalibration.cpp | 6 +++--- .../lightingEstimation/lightingCalibration.hpp | 8 ++++++-- src/software/pipeline/main_lightingCalibration.cpp | 7 +++++-- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/aliceVision/lightingEstimation/lightingCalibration.cpp b/src/aliceVision/lightingEstimation/lightingCalibration.cpp index 0384bc7fd6..154c70d862 100644 --- a/src/aliceVision/lightingEstimation/lightingCalibration.cpp +++ b/src/aliceVision/lightingEstimation/lightingCalibration.cpp @@ -39,7 +39,8 @@ void lightCalibration(const sfmData::SfMData& sfmData, const std::string& inputFile, const std::string& outputPath, const std::string& method, - const bool saveAsModel) + const bool saveAsModel, + const bool ellipticEstimation) { std::vector imageList; std::vector focals; @@ -60,7 +61,6 @@ void lightCalibration(const sfmData::SfMData& sfmData, } } - bool testEllipse = true; bool fromJSON = false; std::vector> allSpheresParams; @@ -180,7 +180,7 @@ void lightCalibration(const sfmData::SfMData& sfmData, { float focal = focals.at(i); std::array sphereParam = allSpheresParams.at(i); - if(testEllipse) + if(ellipticEstimation) { Eigen::Matrix3f K = KMatrices.at(i); float sphereRadius = 1.0; diff --git a/src/aliceVision/lightingEstimation/lightingCalibration.hpp b/src/aliceVision/lightingEstimation/lightingCalibration.hpp index c8e94f3d90..57e809367c 100644 --- a/src/aliceVision/lightingEstimation/lightingCalibration.hpp +++ b/src/aliceVision/lightingEstimation/lightingCalibration.hpp @@ -21,19 +21,23 @@ namespace lightingEstimation { * @param[in] sfmData Input .sfm file to calibrate from * @param[in] inputJSON Path to the JSON file containing the spheres parameters (see sphereDetection) * @param[out] outputPath Path to the JSON file in which lights' directions are written + * @param[in] method Method used for calibration ("brightestPoint" or "whiteSphere") + * @param[in] saveAsModel True to save the estimated lights as model + * @param[in] ellipticEstimation True to use elliptic estimation of the lighting */ void lightCalibration(const sfmData::SfMData& sfmData, const std::string& inputJSON, const std::string& outputPath, const std::string& method, - const bool saveAsModel); + const bool saveAsModel, + const bool ellipticEstimation); /** * @brief Calibrate lighting direction of an image containing a sphere * @param[in] picturePath Path to the image file * @param[in] sphereParam An array of 3 floating-point: the coordinates of the sphere center in the picture frame and the radius of the sphere * @param[in] focal Focal length of the camera - * @param[in] method Method used for calibration (only "brightestPoint" for now) + * @param[in] method Method used for calibration ("brightestPoint" or "whiteSphere") * @param[out] lightingDirection Output parameter for the estimated lighting direction */ void lightCalibrationOneImage(const std::string& picturePath, diff --git a/src/software/pipeline/main_lightingCalibration.cpp b/src/software/pipeline/main_lightingCalibration.cpp index cce465b1dc..03e92b6a0d 100644 --- a/src/software/pipeline/main_lightingCalibration.cpp +++ b/src/software/pipeline/main_lightingCalibration.cpp @@ -52,6 +52,7 @@ int aliceVision_main(int argc, char** argv) std::string ouputJSON; std::string method; bool saveAsModel; + bool ellipticEstimation; // clang-format off po::options_description requiredParams("Required parameters"); @@ -68,7 +69,9 @@ int aliceVision_main(int argc, char** argv) ("saveAsModel, s", po::value(&saveAsModel)->default_value(false), "Calibration used for several datasets.") ("method, m", po::value(&method)->default_value("brightestPoint"), - "Method for light estimation."); + "Method for light estimation.") + ("ellipticEstimation, e", po::value(&ellipticEstimation)->default_value(false), + "Used ellipse model for calibration spheres (more precise)."); // clang-format on CmdLine cmdline("AliceVision lightingCalibration"); @@ -94,7 +97,7 @@ int aliceVision_main(int argc, char** argv) return EXIT_FAILURE; } - lightingEstimation::lightCalibration(sfmData, inputDetection, ouputJSON, method, saveAsModel); + lightingEstimation::lightCalibration(sfmData, inputDetection, ouputJSON, method, saveAsModel, ellipticEstimation); } ALICEVISION_LOG_INFO("Task done in (s): " + std::to_string(timer.elapsed())); From 8fab68c5d9a55a73380913403353c0432c64a097 Mon Sep 17 00:00:00 2001 From: jmelou Date: Wed, 5 Jun 2024 15:40:08 +0200 Subject: [PATCH 57/67] [sphereDetection] Same output for all methods --- src/aliceVision/sphereDetection/sphereDetection.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aliceVision/sphereDetection/sphereDetection.cpp b/src/aliceVision/sphereDetection/sphereDetection.cpp index 83683ca3ff..48ceb3da60 100644 --- a/src/aliceVision/sphereDetection/sphereDetection.cpp +++ b/src/aliceVision/sphereDetection/sphereDetection.cpp @@ -219,7 +219,7 @@ void sphereDetection(const sfmData::SfMData& sfmData, Ort::Session& session, fs: ALICEVISION_LOG_WARNING("No sphere detected for '" << imagePath << "'."); } } - bpt::write_json(outputPath.append("detection.json").string(), fileTree); + bpt::write_json(outputPath.string(), fileTree); } void writeManualSphereJSON(const sfmData::SfMData& sfmData, const std::array& sphereParam, fs::path outputPath) From aca03db4f4f4bdf6d6af055c3d4b0cf86c2aff52 Mon Sep 17 00:00:00 2001 From: jmelou Date: Wed, 5 Jun 2024 15:41:52 +0200 Subject: [PATCH 58/67] [PS] Spelling correction --- .../photometricStereo/photometricStereo.cpp | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/aliceVision/photometricStereo/photometricStereo.cpp b/src/aliceVision/photometricStereo/photometricStereo.cpp index dbd6889e0d..d1f5a28a14 100644 --- a/src/aliceVision/photometricStereo/photometricStereo.cpp +++ b/src/aliceVision/photometricStereo/photometricStereo.cpp @@ -290,7 +290,7 @@ void photometricStereo(const std::vector& imageList, const std::vector>& intList, const Eigen::MatrixXf& lightMat, image::Image& mask, - const std::string& pathToAmbiant, + const std::string& pathToAmbient, const PhotometricSteroParameters& PSParameters, image::Image& normals, image::Image& albedo) @@ -341,19 +341,19 @@ void photometricStereo(const std::vector& imageList, hasMask = true; } - // Read ambiant - image::Image imageAmbiant; + // Read ambient + image::Image imageAmbient; - if (boost::algorithm::icontains(fs::path(pathToAmbiant).stem().string(), "ambiant")) + if (boost::algorithm::icontains(fs::path(pathToAmbient).stem().string(), "ambient")) { - ALICEVISION_LOG_INFO("Removing ambiant light"); - ALICEVISION_LOG_INFO(pathToAmbiant); + ALICEVISION_LOG_INFO("Removing ambient light"); + ALICEVISION_LOG_INFO(pathToAmbient); - image::readImage(pathToAmbiant, imageAmbiant, image::EImageColorSpace::NO_CONVERSION); + image::readImage(pathToAmbient, imageAmbient, image::EImageColorSpace::NO_CONVERSION); if (PSParameters.downscale > 1) { - imageAlgo::resizeImage(PSParameters.downscale, imageAmbiant); + imageAlgo::resizeImage(PSParameters.downscale, imageAmbient); } } @@ -419,9 +419,9 @@ void photometricStereo(const std::vector& imageList, imageAlgo::resizeImage(PSParameters.downscale, imageFloat); } - if (boost::algorithm::icontains(fs::path(pathToAmbiant).stem().string(), "ambiant")) + if (boost::algorithm::icontains(fs::path(pathToAmbient).stem().string(), "ambient")) { - imageFloat = imageFloat - imageAmbiant; + imageFloat = imageFloat - imageAmbient; } intensityScaling(intList.at(i), imageFloat); @@ -620,7 +620,7 @@ void getPicturesNames(const std::string& folderPath, std::vector& i std::transform(fileExtension.begin(), fileExtension.end(), fileExtension.begin(), ::tolower); if (!boost::algorithm::icontains(currentFilePath.stem().string(), "mask") && - !boost::algorithm::icontains(currentFilePath.stem().string(), "ambiant")) + !boost::algorithm::icontains(currentFilePath.stem().string(), "ambient")) { for (const std::string& extension : extensions) { From dd064ef79814eb3a3dc812dc4ffcfc0bb7380213 Mon Sep 17 00:00:00 2001 From: jmelou Date: Fri, 7 Jun 2024 11:53:31 +0200 Subject: [PATCH 59/67] [LightingCalibration] Back to orthographic projection for incident ray --- .../lightingEstimation/lightingCalibration.cpp | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/aliceVision/lightingEstimation/lightingCalibration.cpp b/src/aliceVision/lightingEstimation/lightingCalibration.cpp index 154c70d862..4401939e50 100644 --- a/src/aliceVision/lightingEstimation/lightingCalibration.cpp +++ b/src/aliceVision/lightingEstimation/lightingCalibration.cpp @@ -245,19 +245,23 @@ void lightCalibrationOneImage(const std::string& picturePath, // Observation direction : Eigen::Vector3f observationRayPersp; + Eigen::Vector3f observationRay; // orthographic approximation : - //observationRay(0) = 0.0; - //observationRay(1) = 0.0; - //observationRay(2) = -1.0; + observationRay(0) = 0.0; + observationRay(1) = 0.0; + observationRay(2) = -1.0; + observationRayPersp(0) = brigthestPoint_xy(0) / focal; observationRayPersp(1) = brigthestPoint_xy(1) / focal; observationRayPersp(2) = 1.0; observationRayPersp = -observationRayPersp / observationRayPersp.norm(); // Evaluate lighting direction : - lightingDirection = 2 * normalBrightestPoint.dot(observationRayPersp) * normalBrightestPoint - observationRayPersp; - lightingDirection = lightingDirection / lightingDirection.norm(); + //lightingDirection = 2 * normalBrightestPoint.dot(observationRayPersp) * normalBrightestPoint - observationRayPersp; + //lightingDirection = lightingDirection / lightingDirection.norm(); + + lightingDirection = 2 * normalBrightestPoint.dot(observationRay) * normalBrightestPoint - observationRay; intensity = 1.0; } From 1eb62694e7e6e3aaa81311a57d757244f9378ab4 Mon Sep 17 00:00:00 2001 From: jmelou Date: Fri, 7 Jun 2024 11:53:58 +0200 Subject: [PATCH 60/67] [LightingCalibration] Comments and vrariable names --- src/aliceVision/lightingEstimation/lightingCalibration.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/aliceVision/lightingEstimation/lightingCalibration.cpp b/src/aliceVision/lightingEstimation/lightingCalibration.cpp index 4401939e50..f3db95651a 100644 --- a/src/aliceVision/lightingEstimation/lightingCalibration.cpp +++ b/src/aliceVision/lightingEstimation/lightingCalibration.cpp @@ -551,6 +551,7 @@ void cutImage(const image::Image& imageFloat, image::Image& patch, std::array& patchOrigin) { + // Absolute position of the patch's top left corner in image const int minISphere = floor(sphereParam[1] - sphereParam[2] + imageFloat.rows() / 2); const int minJSphere = floor(sphereParam[0] - sphereParam[2] + imageFloat.cols() / 2); @@ -639,7 +640,7 @@ void writeJSON(const std::string& fileName, } } - for (const auto& [currentTime, viewId] : viewMap) + for (const auto& [currentId, viewId] : viewMap) { const fs::path imagePath = fs::path(viewId.getImage().getImagePath()); From 49a4b13935b3526a06bdcbd9092d07da781039e6 Mon Sep 17 00:00:00 2001 From: jmelou Date: Fri, 7 Jun 2024 11:54:54 +0200 Subject: [PATCH 61/67] [LightingCalibration] Remove case of non-JSON file --- .../lightingCalibration.cpp | 142 ++++++------------ .../pipeline/main_lightingCalibration.cpp | 5 + 2 files changed, 47 insertions(+), 100 deletions(-) diff --git a/src/aliceVision/lightingEstimation/lightingCalibration.cpp b/src/aliceVision/lightingEstimation/lightingCalibration.cpp index f3db95651a..a0752cd414 100644 --- a/src/aliceVision/lightingEstimation/lightingCalibration.cpp +++ b/src/aliceVision/lightingEstimation/lightingCalibration.cpp @@ -61,95 +61,48 @@ void lightCalibration(const sfmData::SfMData& sfmData, } } - bool fromJSON = false; - std::vector> allSpheresParams; std::vector KMatrices; - if(fs::path(inputFile).extension() == ".json") - { - std::cout << "JSON file detected" << std::endl; - fromJSON = true; - } - int nbCols = 0; int nbRows = 0; - if (fromJSON) + // Main tree + bpt::ptree fileTree; + // Read the json file and initialize the tree + bpt::read_json(inputFile, fileTree); + + for (const auto& [currentId, currentView] : viewMap) { - // Main tree - bpt::ptree fileTree; - // Read the json file and initialize the tree - bpt::read_json(inputFile, fileTree); + ALICEVISION_LOG_INFO("View Id: " << currentView.getViewId()); + const fs::path imagePath = fs::path(currentView.getImage().getImagePath()); - for (const auto& [currentTime, currentView] : viewMap) + if (!boost::algorithm::icontains(imagePath.stem().string(), "ambient")) { - ALICEVISION_LOG_INFO("View Id: " << currentView.getViewId()); - const fs::path imagePath = fs::path(currentView.getImage().getImagePath()); - - if (!boost::algorithm::icontains(imagePath.stem().string(), "ambient")) + std::string sphereName = std::to_string(currentView.getViewId()); + auto sphereExists = (fileTree.get_child_optional(sphereName)).is_initialized(); + if (sphereExists) { - std::string sphereName = std::to_string(currentView.getViewId()); - auto sphereExists = (fileTree.get_child_optional(sphereName)).is_initialized(); - if (sphereExists) - { - ALICEVISION_LOG_INFO(" - " << imagePath.string()); - imageList.push_back(imagePath.string()); - - std::array currentSphereParams; - for (auto& currentSphere : fileTree.get_child(sphereName)) - { - currentSphereParams[0] = currentSphere.second.get_child("").get("x", 0.0); - currentSphereParams[1] = currentSphere.second.get_child("").get("y", 0.0); - currentSphereParams[2] = currentSphere.second.get_child("").get("r", 0.0); - } - - allSpheresParams.push_back(currentSphereParams); - - IndexT intrinsicId = currentView.getIntrinsicId(); - focals.push_back(sfmData.getIntrinsics().at(intrinsicId)->getParams().at(0)); - - float focalPx = sfmData.getIntrinsics().at(intrinsicId)->getParams().at(0); - nbCols = sfmData.getIntrinsics().at(intrinsicId)->w(); - nbRows = sfmData.getIntrinsics().at(intrinsicId)->h(); - float x_p = (nbCols) / 2 + sfmData.getIntrinsics().at(intrinsicId)->getParams().at(2); - float y_p = (nbRows) / 2 + sfmData.getIntrinsics().at(intrinsicId)->getParams().at(3); - - Eigen::MatrixXf currentK = Eigen::MatrixXf::Zero(3, 3); - // Create K matrix - currentK << focalPx, 0.0, x_p, - 0.0, focalPx, y_p, - 0.0, 0.0, 1.0; + ALICEVISION_LOG_INFO(" - " << imagePath.string()); + imageList.push_back(imagePath.string()); - KMatrices.push_back(currentK); - } - else + std::array currentSphereParams; + for (auto& currentSphere : fileTree.get_child(sphereName)) { - ALICEVISION_LOG_WARNING("No detected sphere found for '" << imagePath << "'."); + currentSphereParams[0] = currentSphere.second.get_child("").get("x", 0.0); + currentSphereParams[1] = currentSphere.second.get_child("").get("y", 0.0); + currentSphereParams[2] = currentSphere.second.get_child("").get("r", 0.0); } - } - } - } - else - { - IndexT viewId; - for (const auto& [currentTime, currentView] : viewMap) - { - ALICEVISION_LOG_INFO("View Id: " << currentView.getViewId()); - const fs::path imagePath = fs::path(currentView.getImage().getImagePath()); + allSpheresParams.push_back(currentSphereParams); - if (!boost::algorithm::icontains(imagePath.stem().string(), "ambient")) - { - ALICEVISION_LOG_INFO(" - " << imagePath.string()); - imageList.push_back(imagePath.string()); - viewId = currentView.getViewId(); - // Get intrinsics associated with this view : IndexT intrinsicId = currentView.getIntrinsicId(); - const float focalPx = sfmData.getIntrinsics().at(intrinsicId)->getParams().at(0); + focals.push_back(sfmData.getIntrinsics().at(intrinsicId)->getParams().at(0)); + + float focalPx = sfmData.getIntrinsics().at(intrinsicId)->getParams().at(0); nbCols = sfmData.getIntrinsics().at(intrinsicId)->w(); nbRows = sfmData.getIntrinsics().at(intrinsicId)->h(); - const float x_p = (nbCols) / 2 + sfmData.getIntrinsics().at(intrinsicId)->getParams().at(2); - const float y_p = (nbRows) / 2 + sfmData.getIntrinsics().at(intrinsicId)->getParams().at(3); + float x_p = (nbCols) / 2 + sfmData.getIntrinsics().at(intrinsicId)->getParams().at(2); + float y_p = (nbRows) / 2 + sfmData.getIntrinsics().at(intrinsicId)->getParams().at(3); Eigen::MatrixXf currentK = Eigen::MatrixXf::Zero(3, 3); // Create K matrix @@ -159,6 +112,10 @@ void lightCalibration(const sfmData::SfMData& sfmData, KMatrices.push_back(currentK); } + else + { + ALICEVISION_LOG_WARNING("No detected sphere found for '" << imagePath << "'."); + } } } @@ -171,45 +128,30 @@ void lightCalibration(const sfmData::SfMData& sfmData, for (size_t i = 0; i < imageList.size(); ++i) { - std::string picturePath = imageList.at(i); - Eigen::VectorXf lightingDirection = Eigen::VectorXf::Zero(lightSize); float intensity; - if(fromJSON) - { - float focal = focals.at(i); - std::array sphereParam = allSpheresParams.at(i); - if(ellipticEstimation) - { - Eigen::Matrix3f K = KMatrices.at(i); - float sphereRadius = 1.0; - std::array ellipseParam; - cv::Mat maskCV = cv::Mat::zeros(nbRows, nbCols, CV_8UC1); - - image::Image imageFloat; - image::readImage(picturePath, imageFloat, image::EImageColorSpace::NO_CONVERSION); - getEllipseMaskFromSphereParameters(sphereParam, K, ellipseParam, maskCV); - calibrateLightFromRealSphere(imageFloat, maskCV, K, sphereRadius, method, lightingDirection, intensity); - } - else - { - lightCalibrationOneImage(picturePath, sphereParam, focal, method, lightingDirection, intensity); - } - } - else + float focal = focals.at(i); + std::array sphereParam = allSpheresParams.at(i); + if(ellipticEstimation) { Eigen::Matrix3f K = KMatrices.at(i); float sphereRadius = 1.0; - cv::Mat maskCV = cv::imread(inputFile, cv::IMREAD_GRAYSCALE); + std::array ellipseParam; + + cv::Mat maskCV = cv::Mat::zeros(nbRows, nbCols, CV_8UC1); + image::Image imageFloat; image::readImage(picturePath, imageFloat, image::EImageColorSpace::NO_CONVERSION); - + getEllipseMaskFromSphereParameters(sphereParam, K, ellipseParam, maskCV); calibrateLightFromRealSphere(imageFloat, maskCV, K, sphereRadius, method, lightingDirection, intensity); - - break; } + else + { + lightCalibrationOneImage(picturePath, sphereParam, focal, method, lightingDirection, intensity); + } + lightMat.row(i) = lightingDirection; intList.push_back(intensity); } diff --git a/src/software/pipeline/main_lightingCalibration.cpp b/src/software/pipeline/main_lightingCalibration.cpp index 03e92b6a0d..09346c8e1f 100644 --- a/src/software/pipeline/main_lightingCalibration.cpp +++ b/src/software/pipeline/main_lightingCalibration.cpp @@ -88,6 +88,11 @@ int aliceVision_main(int argc, char** argv) ALICEVISION_LOG_ERROR("Directory input: WIP"); ALICEVISION_THROW(std::invalid_argument, "Input directories are not yet supported"); } + else if (fs::path(inputDetection).extension() != ".json") + { + ALICEVISION_LOG_ERROR("The input detection file must be a JSON file."); + ALICEVISION_THROW(std::invalid_argument, "JSON needed for sphere positions and radius."); + } else { sfmData::SfMData sfmData; From 8c2e9d4c9b1c304e529f5c95a3222c59979a4c03 Mon Sep 17 00:00:00 2001 From: jmelou Date: Fri, 7 Jun 2024 11:55:14 +0200 Subject: [PATCH 62/67] [PS] Remove useless code --- src/aliceVision/photometricStereo/photometricStereo.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/aliceVision/photometricStereo/photometricStereo.cpp b/src/aliceVision/photometricStereo/photometricStereo.cpp index d1f5a28a14..b0e660c490 100644 --- a/src/aliceVision/photometricStereo/photometricStereo.cpp +++ b/src/aliceVision/photometricStereo/photometricStereo.cpp @@ -127,11 +127,6 @@ void photometricStereo(const sfmData::SfMData& sfmData, std::map currentMetadata = sfmData.getView(viewId).getImage().getMetadata(); idMap[currentMetadata.at("Exif:DateTimeDigitized")] = viewId; } - - for (const auto& [currentTime, viewId] : idMap) - { - viewIds.push_back(viewId); - } } else { From 4b859aca25a0a917d9898d49bce6bed95aad9020 Mon Sep 17 00:00:00 2001 From: jmelou Date: Wed, 17 Jul 2024 18:09:41 +0200 Subject: [PATCH 63/67] [lightingCalibration] Add debug --- .../lightingEstimation/lightingCalibration.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/aliceVision/lightingEstimation/lightingCalibration.cpp b/src/aliceVision/lightingEstimation/lightingCalibration.cpp index a0752cd414..fe62e5b955 100644 --- a/src/aliceVision/lightingEstimation/lightingCalibration.cpp +++ b/src/aliceVision/lightingEstimation/lightingCalibration.cpp @@ -39,6 +39,7 @@ void lightCalibration(const sfmData::SfMData& sfmData, const std::string& inputFile, const std::string& outputPath, const std::string& method, + const bool doDebug, const bool saveAsModel, const bool ellipticEstimation) { @@ -154,6 +155,13 @@ void lightCalibration(const sfmData::SfMData& sfmData, lightMat.row(i) = lightingDirection; intList.push_back(intensity); + + if(doDebug) + { + int outputSize = 1024; + std::string outputFileName = fs::path(outputPath).parent_path().string() + "/" + fs::path(picturePath).stem().string() + "_" + method + ".png"; + sphereFromLighting(lightingDirection, lightingDirection.norm(), outputFileName, outputSize); + } } // Write in JSON file From 24580ec52d25346c9e7fd9425826606eb6ac48ca Mon Sep 17 00:00:00 2001 From: jmelou Date: Wed, 17 Jul 2024 18:10:11 +0200 Subject: [PATCH 64/67] [lightingCalibration] Be more restrictive on selected pixels --- src/aliceVision/lightingEstimation/lightingCalibration.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/aliceVision/lightingEstimation/lightingCalibration.cpp b/src/aliceVision/lightingEstimation/lightingCalibration.cpp index fe62e5b955..69bf9f1e60 100644 --- a/src/aliceVision/lightingEstimation/lightingCalibration.cpp +++ b/src/aliceVision/lightingEstimation/lightingCalibration.cpp @@ -238,7 +238,7 @@ void lightCalibrationOneImage(const std::string& picturePath, for (int i = 0; i < patch.rows(); ++i) { const float distanceToCenter = std::sqrt((i - radius) * (i - radius) + (j - radius) * (j - radius)); - if ((distanceToCenter < 0.95 * radius) && (patch(i, j) > 0.1) && (patch(i, j) < 0.98)) + if ((distanceToCenter < 0.95 * radius) && (patch(i, j) > 0.2) && (patch(i, j) < 0.8)) { // imSphere = normalSphere.s imSphere(currentIndex) = patch(i, j); @@ -289,7 +289,7 @@ void lightCalibrationOneImage(const std::string& picturePath, for (size_t i = 0; i < patch.rows(); ++i) { float distanceToCenter = sqrt((i - radius) * (i - radius) + (j - radius) * (j - radius)); - if (distanceToCenter < 0.95 * radius && (patch(i, j) > 0.1) && (patch(i, j) < 0.98)) + if (distanceToCenter < 0.95 * radius && (patch(i, j) > 0.2) && (patch(i, j) < 0.8)) { imSphere(currentIndex) = patch(i, j); From 255a31505fd000b65cd4c52cd805bb2ecf8c7913 Mon Sep 17 00:00:00 2001 From: jmelou Date: Wed, 17 Jul 2024 18:10:36 +0200 Subject: [PATCH 65/67] [lightingCalibration] Add debug (hpp) --- src/aliceVision/lightingEstimation/lightingCalibration.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/aliceVision/lightingEstimation/lightingCalibration.hpp b/src/aliceVision/lightingEstimation/lightingCalibration.hpp index 57e809367c..5a24e76a2f 100644 --- a/src/aliceVision/lightingEstimation/lightingCalibration.hpp +++ b/src/aliceVision/lightingEstimation/lightingCalibration.hpp @@ -22,6 +22,7 @@ namespace lightingEstimation { * @param[in] inputJSON Path to the JSON file containing the spheres parameters (see sphereDetection) * @param[out] outputPath Path to the JSON file in which lights' directions are written * @param[in] method Method used for calibration ("brightestPoint" or "whiteSphere") + * @param[in] doDebug True to save debug images * @param[in] saveAsModel True to save the estimated lights as model * @param[in] ellipticEstimation True to use elliptic estimation of the lighting */ @@ -29,6 +30,7 @@ void lightCalibration(const sfmData::SfMData& sfmData, const std::string& inputJSON, const std::string& outputPath, const std::string& method, + const bool doDebug, const bool saveAsModel, const bool ellipticEstimation); From 8dd7a5909e2182b353bbe4cd3d8b482f18f2f320 Mon Sep 17 00:00:00 2001 From: jmelou Date: Wed, 17 Jul 2024 18:12:06 +0200 Subject: [PATCH 66/67] [lightingCalibration] Add debug (main) --- src/software/pipeline/main_lightingCalibration.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/software/pipeline/main_lightingCalibration.cpp b/src/software/pipeline/main_lightingCalibration.cpp index 09346c8e1f..0a5d091fd0 100644 --- a/src/software/pipeline/main_lightingCalibration.cpp +++ b/src/software/pipeline/main_lightingCalibration.cpp @@ -51,6 +51,7 @@ int aliceVision_main(int argc, char** argv) std::string inputDetection; std::string ouputJSON; std::string method; + bool doDebug; bool saveAsModel; bool ellipticEstimation; @@ -70,6 +71,8 @@ int aliceVision_main(int argc, char** argv) "Calibration used for several datasets.") ("method, m", po::value(&method)->default_value("brightestPoint"), "Method for light estimation.") + ("doDebug, d", po::value(&doDebug)->default_value(true), + "Do we save debug images.") ("ellipticEstimation, e", po::value(&ellipticEstimation)->default_value(false), "Used ellipse model for calibration spheres (more precise)."); // clang-format on @@ -102,7 +105,7 @@ int aliceVision_main(int argc, char** argv) return EXIT_FAILURE; } - lightingEstimation::lightCalibration(sfmData, inputDetection, ouputJSON, method, saveAsModel, ellipticEstimation); + lightingEstimation::lightCalibration(sfmData, inputDetection, ouputJSON, method, doDebug, saveAsModel, ellipticEstimation); } ALICEVISION_LOG_INFO("Task done in (s): " + std::to_string(timer.elapsed())); From 977f4cce592795fa2fb4675d7d4dc867190cbc21 Mon Sep 17 00:00:00 2001 From: jmelou Date: Wed, 17 Jul 2024 18:20:36 +0200 Subject: [PATCH 67/67] [photometricStereo] Add outputs --- .../photometricStereo/photometricStereo.cpp | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/aliceVision/photometricStereo/photometricStereo.cpp b/src/aliceVision/photometricStereo/photometricStereo.cpp index b0e660c490..2a3cc1c7a6 100644 --- a/src/aliceVision/photometricStereo/photometricStereo.cpp +++ b/src/aliceVision/photometricStereo/photometricStereo.cpp @@ -223,6 +223,14 @@ void photometricStereo(const sfmData::SfMData& sfmData, outputPath + "/" + std::to_string(posesIt.first) + "_normals_w.exr", normals, image::ImageWriteOptions().toColorSpace(image::EImageColorSpace::NO_CONVERSION).storageDataType(image::EStorageDataType::Float)); + + + image::Image normalsImPNG(normals.cols(), normals.rows()); + convertNormalMap2png(normals, normalsImPNG); + image::writeImage( + outputPath + "/" + std::to_string(posesIt.first) + "_normals_w.png", + normalsImPNG, + image::ImageWriteOptions().toColorSpace(image::EImageColorSpace::NO_CONVERSION).storageDataType(image::EStorageDataType::Float)); } if (skipAll) @@ -279,6 +287,21 @@ void photometricStereo(const sfmData::SfMData& sfmData, sfmDataIO::save( normalSfmData, outputPath + "/normalMaps.sfm", sfmDataIO::ESfMData(sfmDataIO::VIEWS | sfmDataIO::INTRINSICS | sfmDataIO::EXTRINSICS)); + + // Create Normal_png SfmData + sfmData::SfMData normalSfmDataPNG = albedoSfmData; + for (auto& viewIt : normalSfmDataPNG.getViews()) + { + const IndexT viewId = viewIt.first; + IndexT poseId = viewIt.second->getPoseId(); + + sfmData::View* view = normalSfmDataPNG.getViews().at(viewId).get(); + std::string imagePath = outputPath + "/" + std::to_string(poseId) + "_normals_w.png"; + view->getImage().setImagePath(imagePath); + } + + sfmDataIO::save( + normalSfmDataPNG, outputPath + "/normalMapsPNG.sfm", sfmDataIO::ESfMData(sfmDataIO::VIEWS | sfmDataIO::INTRINSICS | sfmDataIO::EXTRINSICS)); } void photometricStereo(const std::vector& imageList,