diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c8e5ea5861..863c5a7d76 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -51,7 +51,7 @@ option(ALICEVISION_BUILD_SFM "Build AliceVision SfM part" ON) option(ALICEVISION_BUILD_MVS "Build AliceVision MVS part" ON) option(ALICEVISION_BUILD_HDR "Build AliceVision HDR part" ON) option(ALICEVISION_BUILD_SEGMENTATION "Build AliceVision Segmentation part" ON) -option(ALICEVISION_BUILD_STEREOPHOTOMETRY "Build AliceVision StereoPhotometry part" ON) +option(ALICEVISION_BUILD_PHOTOMETRICSTEREO "Build AliceVision Photometric Stereo part" ON) option(ALICEVISION_BUILD_PANORAMA "Build AliceVision panorama part" ON) option(ALICEVISION_BUILD_SOFTWARE "Build AliceVision command line tools." ON) option(ALICEVISION_BUILD_COVERAGE "Enable code coverage generation (gcc only)" OFF) @@ -94,7 +94,7 @@ if(NOT ALICEVISION_BUILD_SFM) SET(ALICEVISION_BUILD_MVS OFF) SET(ALICEVISION_BUILD_HDR OFF) SET(ALICEVISION_BUILD_SEGMENTATION OFF) - SET(ALICEVISION_BUILD_STEREOPHOTOMETRY OFF) + SET(ALICEVISION_BUILD_PHOTOMETRICSTEREO OFF) SET(ALICEVISION_BUILD_PANORAMA OFF) SET(ALICEVISION_BUILD_LIDAR OFF) endif() diff --git a/src/aliceVision/CMakeLists.txt b/src/aliceVision/CMakeLists.txt index fbbdfaf3e0..9e5dd9a62c 100644 --- a/src/aliceVision/CMakeLists.txt +++ b/src/aliceVision/CMakeLists.txt @@ -51,7 +51,6 @@ endif() # MVS modules if(ALICEVISION_BUILD_MVS) - add_subdirectory(lightingEstimation) add_subdirectory(mesh) add_subdirectory(mvsData) add_subdirectory(mvsUtils) @@ -72,12 +71,13 @@ if(ALICEVISION_BUILD_SFM AND ALICEVISION_BUILD_MVS) add_subdirectory(sfmMvsUtils) endif() -if (ALICEVISION_BUILD_STEREOPHOTOMETRY) +if (ALICEVISION_BUILD_PHOTOMETRICSTEREO) if(ALICEVISION_HAVE_OPENCV) add_subdirectory(photometricStereo) if(ALICEVISION_HAVE_ONNX) add_subdirectory(sphereDetection) endif() + add_subdirectory(lightingEstimation) endif() endif() diff --git a/src/aliceVision/fuseCut/DelaunayGraphCut_test.cpp b/src/aliceVision/fuseCut/DelaunayGraphCut_test.cpp index 58c1e110f2..5ce366cccd 100644 --- a/src/aliceVision/fuseCut/DelaunayGraphCut_test.cpp +++ b/src/aliceVision/fuseCut/DelaunayGraphCut_test.cpp @@ -67,7 +67,7 @@ BOOST_AUTO_TEST_CASE(fuseCut_delaunayGraphCut) const NViewDatasetConfigurator config(1000, 1000, 500, 500, 1, 0); SfMData sfmData = generateSfm(config, 6); - mvsUtils::MultiViewParams mp(sfmData, "", "", "", false); + mvsUtils::MultiViewParams mp(sfmData, "", "", ""); mp.userParams.put("LargeScale.universePercentile", 0.999); mp.userParams.put("delaunaycut.seed", 1); diff --git a/src/aliceVision/lightingEstimation/CMakeLists.txt b/src/aliceVision/lightingEstimation/CMakeLists.txt index 34297a7a07..011d7ef881 100644 --- a/src/aliceVision/lightingEstimation/CMakeLists.txt +++ b/src/aliceVision/lightingEstimation/CMakeLists.txt @@ -3,6 +3,7 @@ set(lightingEstimation_files_headers augmentedNormals.hpp lightingEstimation.hpp lightingCalibration.hpp + ellipseGeometry.hpp ) # Sources @@ -10,13 +11,16 @@ set(lightingEstimation_files_sources augmentedNormals.cpp lightingEstimation.cpp lightingCalibration.cpp + ellipseGeometry.cpp ) alicevision_add_library(aliceVision_lightingEstimation SOURCES ${lightingEstimation_files_headers} ${lightingEstimation_files_sources} PUBLIC_LINKS + ${OpenCV_LIBS} aliceVision_image aliceVision_system + aliceVision_photometricStereo ) diff --git a/src/aliceVision/lightingEstimation/augmentedNormals.hpp b/src/aliceVision/lightingEstimation/augmentedNormals.hpp index e98bba686e..4eb7de4bba 100644 --- a/src/aliceVision/lightingEstimation/augmentedNormals.hpp +++ b/src/aliceVision/lightingEstimation/augmentedNormals.hpp @@ -67,8 +67,8 @@ class AugmentedNormal : public Eigen::Matrix inline const T& nz() const { return (*this)(2); } inline T& nz() { return (*this)(2); } - inline const T& nambiant() const { return (*this)(3); } - inline T& nambiant() { return (*this)(3); } + inline const T& nambient() const { return (*this)(3); } + inline T& nambient() { return (*this)(3); } inline const T& nx_ny() const { return (*this)(4); } inline T& nx_ny() { return (*this)(4); } diff --git a/src/aliceVision/lightingEstimation/ellipseGeometry.cpp b/src/aliceVision/lightingEstimation/ellipseGeometry.cpp new file mode 100644 index 0000000000..20a2bbcc7d --- /dev/null +++ b/src/aliceVision/lightingEstimation/ellipseGeometry.cpp @@ -0,0 +1,232 @@ +// This file is part of the AliceVision project. +// Copyright (c) 2023 AliceVision contributors. +// This Source Code Form is subject to the terms of the Mozilla Public License, +// v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +#include "ellipseGeometry.hpp" +#include +#include + +#include +#include + +#include +#include + +namespace aliceVision { +namespace lightingEstimation { + +#define PI 3.14159265 + +void quadraticFromEllipseParameters(const std::array& ellipseParameters, + Eigen::Matrix3f& Q) +{ + float phi = ellipseParameters[0]*PI/180.0; + float c_x = ellipseParameters[1]; + float c_y = ellipseParameters[2]; + float b = ellipseParameters[3]; + float a = ellipseParameters[4]; + + float A = a*a*sin(phi)*sin(phi) + b*b*cos(phi)*cos(phi); + float B = 2*(b*b-a*a)*sin(phi)*cos(phi); + float C = a*a*cos(phi)*cos(phi) + b*b*sin(phi)*sin(phi); + float D = -2*A*c_x - B*c_y; + float E = -B*c_x - 2*C*c_y; + float F = A*c_x*c_x + C*c_y*c_y - a*a*b*b + B*c_x*c_y; + + Q << A, B/2, D/2, + B/2, C, E/2, + D/2, E/2, F; + + Q /= Q.norm(); +} + +int findUniqueIndex(const std::vector& vec) { + + std::unordered_map countMap; + + // Compter le nombre d'occurrences de chaque élément dans le vecteur + for (int num : vec) { + countMap[num]++; + } + + int toReturn = -1; + // Trouver l'élément avec une seule occurrence + for (size_t i = 0; i < vec.size(); ++i) { + if (countMap[vec[i]] == 1) { + toReturn = i; + } + } + + return toReturn; +} + +void estimateSphereCenter(const std::array& ellipseParameters, + const float sphereRadius, + const Eigen::Matrix3f& K, + std::array& sphereCenter) +{ + Eigen::Matrix3f Q; + quadraticFromEllipseParameters(ellipseParameters, Q); + + Eigen::Matrix3f M = K.transpose()*Q*K; + Eigen::EigenSolver es(M); + + Eigen::Vector3f eigval = es.eigenvalues().real(); + + std::vector eigvalSign; + + for (int i = 0; i < 3; ++i) { + eigvalSign.push_back((eigval[i] > 0) ? 1 : -1); + } + + int index = findUniqueIndex(eigvalSign); + float uniqueEigval = eigval[index]; + + float dist = sqrt(1 - ((eigval[0] + eigval[1] + eigval[2] - uniqueEigval)/2) / uniqueEigval); + + Eigen::Vector3f eigvect = es.eigenvectors().col(index).real(); + float sign = eigvect[2] > 0 ? 1 : -1; + + float norm = eigvect.norm(); + float C_factor = sphereRadius*dist*sign/norm; + Eigen::Vector3f C = C_factor*eigvect; + sphereCenter[0]=C[0]; + sphereCenter[1]=C[1]; + sphereCenter[2]=C[2]; +} + +void sphereRayIntersection(const Eigen::Vector3f& direction, + const std::array& sphereCenter, + const float sphereRadius, + float& delta, + Eigen::Vector3f& normal) +{ + float a = direction.dot(direction); + + Eigen::Vector3f spCenter; + spCenter << sphereCenter[0], sphereCenter[1], sphereCenter[2]; + + float b = -2*direction.dot(spCenter); + float c = spCenter.dot(spCenter) - sphereRadius*sphereRadius; + + delta = b*b - 4*a*c; + + float factor; + + if (delta >= 0) { + factor = (-b - sqrt(delta))/(2*a); + normal = (direction*factor - spCenter)/sphereRadius; + } + else { + delta = -1; + } +} + +void estimateSphereNormals(const std::array& sphereCenter, + const float sphereRadius, + const Eigen::Matrix3f& K, + image::Image& normals, + image::Image& newMask) +{ + Eigen::Matrix3f invK = K.inverse(); + + for (int i = 0; i < normals.rows(); ++i) { + for (int j = 0; j < normals.cols(); ++j) { + // Get homogeneous coordinates of the pixel : + Eigen::Vector3f coordinates = Eigen::Vector3f(j, i, 1.0f); + + // Get the direction of the ray : + Eigen::Vector3f direction = invK*coordinates; + + // Estimate the interception of the ray with the sphere : + float delta = 0.0; + Eigen::Vector3f normal = Eigen::Vector3f(0.0f, 0.0f, 0.0f); + + sphereRayIntersection(direction, sphereCenter, sphereRadius, delta, normal); + + // If the ray intercepts the sphere we add this pixel to the new mask : + if(delta > 0) + { + newMask(i, j) = 1.0; + normals(i, j) = image::RGBfColor(normal[0], normal[1], normal[2]); + } + else + { + newMask(i, j) = 0.0; + normals(i, j) = image::RGBfColor(0.0, 0.0, 0.0); + } + } + } +} + +void getRealNormalOnSphere(const cv::Mat& maskCV, + const Eigen::Matrix3f& K, + const float sphereRadius, + image::Image& normals, + image::Image& newMask) +{ + + // Apply a threshold to the image + int thresholdValue = 150; + cv::Mat thresh; + cv::threshold(maskCV, thresh, thresholdValue, 255, 0); + + // Find contours + std::vector> contours; + std::vector hierarchy; + cv::findContours(thresh, contours, hierarchy, cv::RETR_TREE, cv::CHAIN_APPROX_SIMPLE); + + // Fit the ellipse to the first contour + std::vector cnt = contours[0]; + cv::RotatedRect ellipse = cv::fitEllipse(cnt); + float angle = ellipse.angle; + cv::Size2f size = ellipse.size; + cv::Point2f center = ellipse.center; + + // Ellipse is converted as five-parameter array + std::array ellipseParameters; + + ellipseParameters[0] = angle; + ellipseParameters[1] = center.x; + ellipseParameters[2] = center.y; + ellipseParameters[3] = size.width/2; + ellipseParameters[4] = size.height/2; + + std::array sphereCenter; + estimateSphereCenter(ellipseParameters, sphereRadius, K, sphereCenter); + estimateSphereNormals(sphereCenter, sphereRadius, K, normals, newMask); +} + +void getEllipseMaskFromSphereParameters(const std::array& sphereParam, const Eigen::Matrix3f& K, std::array& ellipseParameters, cv::Mat maskCV) +{ + + // Distance between center of image and center of disk : + float imX = K(0, 2); + float imY = K(1, 2); + float f = K(0, 0); + float delta = sqrt((sphereParam[0])*(sphereParam[0]) + (sphereParam[1])*(sphereParam[1])); + // sphere params = x,y, radius + // ellipse params = angle, x, y, semi minor axe, semi major axe + + // Ellipse from sphere parameters + // semi minor axe = radius; + // semi major axe = formula from paper + // main direction = disc center to picture center + // ellipse center = disc center + + float radians = atan((sphereParam[0]) / (sphereParam[1])); + ellipseParameters[0] = radians * (180.0/3.141592653589793238463); + + ellipseParameters[1] = sphereParam[0]; + ellipseParameters[2] = sphereParam[1]; + ellipseParameters[3] = sphereParam[2]; + + + // a² = b² ((distance between image center and disc center)² + f² + b²)/(f² + b²) + ellipseParameters[4] = sqrt((ellipseParameters[3]*ellipseParameters[3]) * (delta * delta + f*f + ellipseParameters[3]*ellipseParameters[3])/(f*f + ellipseParameters[3]*ellipseParameters[3])); +} + +} // namespace lightingEstimation +} // namespace aliceVision \ No newline at end of file diff --git a/src/aliceVision/lightingEstimation/ellipseGeometry.hpp b/src/aliceVision/lightingEstimation/ellipseGeometry.hpp new file mode 100644 index 0000000000..954e293ea3 --- /dev/null +++ b/src/aliceVision/lightingEstimation/ellipseGeometry.hpp @@ -0,0 +1,75 @@ +// This file is part of the AliceVision project. +// Copyright (c) 2024 AliceVision contributors. +// This Source Code Form is subject to the terms of the Mozilla Public License, +// v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +#pragma once + +#include + +#include +#include + +#include +#include +#include + +namespace aliceVision { +namespace lightingEstimation { + +/** + * @brief Estimate the center of a sphere from the ellipse parameters + * @param[in] ellipseParameters An array of 5 floating-point: the parameters of the ellipse + * @param[in] sphereRadius The radius of the sphere + * @param[in] K Intrinsic parameters of the camera + * @param[out] sphereCenter An array of 3 floating-point: the coordinates of the sphere center in the picture frame + */ +void estimateSphereCenter(const std::array& ellipseParameters, + const float sphereRadius, + const Eigen::Matrix3f& K, + std::array& sphereCenter); + +void sphereRayIntersection(const Eigen::Vector3f& direction, + const std::array& sphereCenter, + const float sphereRadius, + float& delta, + Eigen::Vector3f& normal); + +void quadraticFromEllipseParameters(const std::array& ellipseParameters, + Eigen::Matrix3f& Q); + +int findUniqueIndex(const std::vector& vec); + +/** + * @brief Estimate the normals of a sphere from a mask + * @param[in] sphereCenter An array of 3 floating-point: the coordinates of the sphere center in the picture frame + * @param[in] ellipseMask The binary mask of the sphere in the picture + * @param[in] K Intrinsic parameters of the camera + * @param[out] normals Normals on the sphere + * @param[out] newMask The mask of the sphere after ray tracing + */ +void estimateSphereNormals(const std::array& sphereCenter, + const float sphereRadius, + const Eigen::Matrix3f& K, + image::Image& normals, + image::Image& newMask); + +/** + * @brief Estimate the normals of a sphere from a mask + * @param[in] maskCV The openCV image of the binary mask of the ellipse in the picture + * @param[in] K Intrinsic parameters of the camera + * @param[out] normals Normals on the sphere in camera frame + * @param[out] newMask The mask of the sphere after ray tracing +*/ +void getRealNormalOnSphere(const cv::Mat& maskCV, + const Eigen::Matrix3f& K, + const float sphereRadius, + image::Image& normals, + image::Image& newMask); + + +void getEllipseMaskFromSphereParameters(const std::array& sphereParam, const Eigen::Matrix3f& K, std::array& ellipseParameters, cv::Mat maskCV); + +} // namespace lightingEstimation +} // namespace aliceVision \ No newline at end of file diff --git a/src/aliceVision/lightingEstimation/lightingCalibration.cpp b/src/aliceVision/lightingEstimation/lightingCalibration.cpp index 4b8ca502c3..69bf9f1e60 100644 --- a/src/aliceVision/lightingEstimation/lightingCalibration.cpp +++ b/src/aliceVision/lightingEstimation/lightingCalibration.cpp @@ -6,11 +6,13 @@ #include "lightingCalibration.hpp" #include "lightingEstimation.hpp" +#include "ellipseGeometry.hpp" #include #include #include #include +#include #include @@ -34,35 +36,49 @@ namespace aliceVision { namespace lightingEstimation { void lightCalibration(const sfmData::SfMData& sfmData, - const std::string& inputJSON, + const std::string& inputFile, const std::string& outputPath, const std::string& method, - const bool saveAsModel) + const bool doDebug, + const bool saveAsModel, + const bool ellipticEstimation) { std::vector imageList; - std::vector> allSpheresParams; std::vector focals; - std::string inputJSONFullName = inputJSON + "/detection.json"; - - // Main tree - bpt::ptree fileTree; - // Read the json file and initialize the tree - bpt::read_json(inputJSONFullName, fileTree); - std::map viewMap; for (auto& viewIt : sfmData.getViews()) { std::map currentMetadata = sfmData.getView(viewIt.first).getImage().getMetadata(); - viewMap[currentMetadata.at("Exif:DateTimeDigitized")] = sfmData.getView(viewIt.first); + + if (currentMetadata.find("Exif:DateTimeDigitized") == currentMetadata.end()) + { + std::cout << "No metadata case" << std::endl; + viewMap[sfmData.getView(viewIt.first).getImage().getImagePath()] = sfmData.getView(viewIt.first); + } + else + { + viewMap[currentMetadata.at("Exif:DateTimeDigitized")] = sfmData.getView(viewIt.first); + } } - for (const auto& [currentTime, currentView] : viewMap) + std::vector> allSpheresParams; + std::vector KMatrices; + + int nbCols = 0; + int nbRows = 0; + + // Main tree + bpt::ptree fileTree; + // Read the json file and initialize the tree + bpt::read_json(inputFile, fileTree); + + for (const auto& [currentId, currentView] : viewMap) { ALICEVISION_LOG_INFO("View Id: " << currentView.getViewId()); const fs::path imagePath = fs::path(currentView.getImage().getImagePath()); - if (!boost::algorithm::icontains(imagePath.stem().string(), "ambiant")) + if (!boost::algorithm::icontains(imagePath.stem().string(), "ambient")) { std::string sphereName = std::to_string(currentView.getViewId()); auto sphereExists = (fileTree.get_child_optional(sphereName)).is_initialized(); @@ -72,18 +88,30 @@ void lightCalibration(const sfmData::SfMData& sfmData, imageList.push_back(imagePath.string()); std::array currentSphereParams; - for (auto& currentSphere : fileTree.get_child(sphereName)) { currentSphereParams[0] = currentSphere.second.get_child("").get("x", 0.0); currentSphereParams[1] = currentSphere.second.get_child("").get("y", 0.0); currentSphereParams[2] = currentSphere.second.get_child("").get("r", 0.0); } - allSpheresParams.push_back(currentSphereParams); IndexT intrinsicId = currentView.getIntrinsicId(); focals.push_back(sfmData.getIntrinsics().at(intrinsicId)->getParams().at(0)); + + float focalPx = sfmData.getIntrinsics().at(intrinsicId)->getParams().at(0); + nbCols = sfmData.getIntrinsics().at(intrinsicId)->w(); + nbRows = sfmData.getIntrinsics().at(intrinsicId)->h(); + float x_p = (nbCols) / 2 + sfmData.getIntrinsics().at(intrinsicId)->getParams().at(2); + float y_p = (nbRows) / 2 + sfmData.getIntrinsics().at(intrinsicId)->getParams().at(3); + + Eigen::MatrixXf currentK = Eigen::MatrixXf::Zero(3, 3); + // Create K matrix + currentK << focalPx, 0.0, x_p, + 0.0, focalPx, y_p, + 0.0, 0.0, 1.0; + + KMatrices.push_back(currentK); } else { @@ -92,30 +120,60 @@ void lightCalibration(const sfmData::SfMData& sfmData, } } - Eigen::MatrixXf lightMat(imageList.size(), 3); + int lightSize = 3; + if (!method.compare("SH")) + lightSize = 9; + + Eigen::MatrixXf lightMat(imageList.size(), lightSize); std::vector intList; for (size_t i = 0; i < imageList.size(); ++i) { std::string picturePath = imageList.at(i); - std::array sphereParam = allSpheresParams.at(i); + Eigen::VectorXf lightingDirection = Eigen::VectorXf::Zero(lightSize); + float intensity; + float focal = focals.at(i); + std::array sphereParam = allSpheresParams.at(i); + if(ellipticEstimation) + { + Eigen::Matrix3f K = KMatrices.at(i); + float sphereRadius = 1.0; + std::array ellipseParam; + + cv::Mat maskCV = cv::Mat::zeros(nbRows, nbCols, CV_8UC1); + + image::Image imageFloat; + image::readImage(picturePath, imageFloat, image::EImageColorSpace::NO_CONVERSION); + getEllipseMaskFromSphereParameters(sphereParam, K, ellipseParam, maskCV); + calibrateLightFromRealSphere(imageFloat, maskCV, K, sphereRadius, method, lightingDirection, intensity); + } + else + { + lightCalibrationOneImage(picturePath, sphereParam, focal, method, lightingDirection, intensity); + } - Eigen::Vector3f lightingDirection; - lightCalibrationOneImage(picturePath, sphereParam, focal, method, lightingDirection); lightMat.row(i) = lightingDirection; - intList.push_back(lightingDirection.norm()); + intList.push_back(intensity); + + if(doDebug) + { + int outputSize = 1024; + std::string outputFileName = fs::path(outputPath).parent_path().string() + "/" + fs::path(picturePath).stem().string() + "_" + method + ".png"; + sphereFromLighting(lightingDirection, lightingDirection.norm(), outputFileName, outputSize); + } } // Write in JSON file - writeJSON(outputPath, sfmData, imageList, lightMat, intList, saveAsModel); + writeJSON(outputPath, sfmData, imageList, lightMat, intList, saveAsModel, method); } void lightCalibrationOneImage(const std::string& picturePath, const std::array& sphereParam, const float focal, const std::string& method, - Eigen::Vector3f& lightingDirection) + Eigen::VectorXf& lightingDirection, + float& intensity) { // Read picture : image::Image imageFloat; @@ -128,10 +186,15 @@ void lightCalibrationOneImage(const std::string& picturePath, Eigen::Vector2f brigthestPoint; detectBrightestPoint(sphereParam, imageFloat, brigthestPoint); + Eigen::Vector2f brigthestPoint_xy; + brigthestPoint_xy(0) = brigthestPoint(0) - imageFloat.cols() / 2; + brigthestPoint_xy(1) = brigthestPoint(1) - imageFloat.rows() / 2; + Eigen::Vector3f normalBrightestPoint; - getNormalOnSphere(brigthestPoint(0), brigthestPoint(1), sphereParam, normalBrightestPoint); + getNormalOnSphere(brigthestPoint_xy(0), brigthestPoint_xy(1), sphereParam, normalBrightestPoint); // Observation direction : + Eigen::Vector3f observationRayPersp; Eigen::Vector3f observationRay; // orthographic approximation : @@ -139,11 +202,20 @@ void lightCalibrationOneImage(const std::string& picturePath, observationRay(1) = 0.0; observationRay(2) = -1.0; + observationRayPersp(0) = brigthestPoint_xy(0) / focal; + observationRayPersp(1) = brigthestPoint_xy(1) / focal; + observationRayPersp(2) = 1.0; + observationRayPersp = -observationRayPersp / observationRayPersp.norm(); + // Evaluate lighting direction : + //lightingDirection = 2 * normalBrightestPoint.dot(observationRayPersp) * normalBrightestPoint - observationRayPersp; + //lightingDirection = lightingDirection / lightingDirection.norm(); + lightingDirection = 2 * normalBrightestPoint.dot(observationRay) * normalBrightestPoint - observationRay; - lightingDirection = lightingDirection / lightingDirection.norm(); + + intensity = 1.0; } - // If method = HS : + // If method = whiteSphere : else if (!method.compare("whiteSphere")) { // Evaluate light direction and intensity by pseudo-inverse @@ -165,8 +237,8 @@ void lightCalibrationOneImage(const std::string& picturePath, { for (int i = 0; i < patch.rows(); ++i) { - float distanceToCenter = (i - radius) * (i - radius) + (j - radius) * (j - radius); - if ((distanceToCenter < (radius * radius - 0.05 * radius)) && (patch(i, j) > 0.3) && (patch(i, j) < 0.8)) + const float distanceToCenter = std::sqrt((i - radius) * (i - radius) + (j - radius) * (j - radius)); + if ((distanceToCenter < 0.95 * radius) && (patch(i, j) > 0.2) && (patch(i, j) < 0.8)) { // imSphere = normalSphere.s imSphere(currentIndex) = patch(i, j); @@ -180,12 +252,177 @@ void lightCalibrationOneImage(const std::string& picturePath, } } } + Eigen::MatrixXf normalSphereMasked(currentIndex, 3); normalSphereMasked = normalSphere.block(0, 0, currentIndex, 3); Eigen::VectorXf imSphereMasked(currentIndex); imSphereMasked = imSphere.head(currentIndex); lightingDirection = normalSphere.colPivHouseholderQr().solve(imSphere); + + intensity = lightingDirection.norm(); + lightingDirection = lightingDirection / intensity; + } + + // If method = SH : + else if (!method.compare("SH")) + { + size_t lightSize = lightingDirection.size(); + + // Evaluate light direction and intensity by pseudo-inverse + int minISphere = floor(sphereParam[1] - sphereParam[2] + imageFloat.rows() / 2); + int minJSphere = floor(sphereParam[0] - sphereParam[2] + imageFloat.cols() / 2); + + float radius = sphereParam[2]; + + image::Image patch; + patch = imageFloat.block(minISphere, minJSphere, 2 * radius, 2 * radius); + + int nbPixelsPatch = 4 * radius * radius; + Eigen::VectorXf imSphere(nbPixelsPatch); + Eigen::MatrixXf normalSphere(nbPixelsPatch, lightSize); + + int currentIndex = 0; + + for (size_t j = 0; j < patch.cols(); ++j) + { + for (size_t i = 0; i < patch.rows(); ++i) + { + float distanceToCenter = sqrt((i - radius) * (i - radius) + (j - radius) * (j - radius)); + if (distanceToCenter < 0.95 * radius && (patch(i, j) > 0.2) && (patch(i, j) < 0.8)) + { + imSphere(currentIndex) = patch(i, j); + + normalSphere(currentIndex, 0) = (float(j) - radius) / radius; + normalSphere(currentIndex, 1) = (float(i) - radius) / radius; + normalSphere(currentIndex, 2) = -sqrt(1 - normalSphere(currentIndex, 0) * normalSphere(currentIndex, 0) - + normalSphere(currentIndex, 1) * normalSphere(currentIndex, 1)); + normalSphere(currentIndex, 3) = 1; + if (lightSize > 4) + { + normalSphere(currentIndex, 4) = normalSphere(currentIndex, 0) * normalSphere(currentIndex, 1); + normalSphere(currentIndex, 5) = normalSphere(currentIndex, 0) * normalSphere(currentIndex, 2); + normalSphere(currentIndex, 6) = normalSphere(currentIndex, 1) * normalSphere(currentIndex, 2); + normalSphere(currentIndex, 7) = normalSphere(currentIndex, 0) * normalSphere(currentIndex, 0) - + normalSphere(currentIndex, 1) * normalSphere(currentIndex, 1); + normalSphere(currentIndex, 8) = 3 * normalSphere(currentIndex, 2) * normalSphere(currentIndex, 2) - 1; + } + ++currentIndex; + } + } + } + + Eigen::MatrixXf normalSphereMasked(currentIndex, lightSize); + normalSphereMasked = normalSphere.block(0, 0, currentIndex, lightSize); + + Eigen::VectorXf imSphereMasked(currentIndex); + imSphereMasked = imSphere.head(currentIndex); + + // 1) Directionnal part estimation : + Eigen::MatrixXf normalOrdre1(currentIndex, 3); + normalOrdre1 = normalSphereMasked.leftCols(3); + Eigen::Vector3f directionnalPart = normalOrdre1.colPivHouseholderQr().solve(imSphereMasked); + intensity = directionnalPart.norm(); + directionnalPart = directionnalPart / intensity; + + // 2) Other order estimation : + Eigen::VectorXf imSphereModif(currentIndex); + imSphereModif = imSphereMasked; + for (size_t i = 0; i < currentIndex; ++i) + { + for (size_t k = 0; k < 3; ++k) + { + imSphereModif(i) -= normalSphereMasked(i, k) * directionnalPart(k); + } + } + Eigen::VectorXf secondOrder(6); + secondOrder = normalSphereMasked.rightCols(6).colPivHouseholderQr().solve(imSphereModif); + + lightingDirection << directionnalPart, secondOrder; + } +} + +void calibrateLightFromRealSphere(const image::Image& imageFloat, + const cv::Mat& maskCV, + const Eigen::Matrix3f& K, + const float sphereRadius, + const std::string& method, + Eigen::VectorXf& lightingDirection, + float& intensity) +{ + image::Image normals(imageFloat.width(), imageFloat.height()); + image::Image newMask(imageFloat.width(), imageFloat.height()); + + getRealNormalOnSphere(maskCV, K, sphereRadius, normals, newMask); + + image::Image normalsPNG(maskCV.cols, maskCV.rows); + aliceVision::photometricStereo::convertNormalMap2png(normals, normalsPNG); + + // If method = brightest point : + if (!method.compare("brightestPoint")) + { + // Detect brightest point : + Eigen::Vector2f brigthestPoint; + detectBrightestPoint(newMask, imageFloat, brigthestPoint); + + Eigen::Vector2f brigthestPoint_xy; + brigthestPoint_xy(0) = brigthestPoint(0) - imageFloat.cols() / 2; + brigthestPoint_xy(1) = brigthestPoint(1) - imageFloat.rows() / 2; + + Eigen::Vector3f normalBrightestPoint; + normalBrightestPoint = normals(round(brigthestPoint(1)), round(brigthestPoint(0))).cast(); + + // Observation direction : + Eigen::Vector3f observationRayPersp; + + // orthographic approximation : + observationRayPersp(0) = brigthestPoint_xy(0) / K(0,0); + observationRayPersp(1) = brigthestPoint_xy(1) / K(0,0); + observationRayPersp(2) = 1.0; + observationRayPersp = -observationRayPersp / observationRayPersp.norm(); + + // Evaluate lighting direction : + lightingDirection = 2 * normalBrightestPoint.dot(observationRayPersp) * normalBrightestPoint - observationRayPersp; + lightingDirection = lightingDirection / lightingDirection.norm(); + + intensity = 1.0; + } + + // If method = whiteSphere : + else if (!method.compare("whiteSphere")) + { + // Evaluate light direction and intensity by pseudo-inverse + + std::vector indices; + aliceVision::photometricStereo::getIndMask(newMask, indices); + + Eigen::VectorXf imSphere(indices.size()); + Eigen::MatrixXf normalSphere(indices.size(), 3); + + int currentIndex = 0; + + for (int j = 0; j < newMask.cols(); ++j) + { + for (int i = 0; i < newMask.rows(); ++i) + { + if (newMask(i,j) > 0.1 && (imageFloat(i, j) > 0.2) && (imageFloat(i, j) < 0.8)) + { + // imSphere = normalSphere.s + imSphere(currentIndex) = imageFloat(i, j); + + normalSphere(currentIndex, 0) = normals(i,j)(0); + normalSphere(currentIndex, 1) = normals(i,j)(1); + normalSphere(currentIndex, 2) = normals(i,j)(2); + + ++currentIndex; + } + } + } + + lightingDirection = normalSphere.colPivHouseholderQr().solve(imSphere); + + intensity = lightingDirection.norm(); + lightingDirection = lightingDirection / intensity; } } @@ -209,8 +446,32 @@ void detectBrightestPoint(const std::array& sphereParam, const image:: Eigen::Index maxRow, maxCol; static_cast(convolutedPatch2.maxCoeff(&maxRow, &maxCol)); - brigthestPoint(0) = maxCol + patchOrigin[0] - imageFloat.cols() / 2; - brigthestPoint(1) = maxRow + patchOrigin[1] - imageFloat.rows() / 2; + brigthestPoint(0) = maxCol + patchOrigin[0]; + brigthestPoint(1) = maxRow + patchOrigin[1]; +} + +void detectBrightestPoint(const image::Image newMask, const image::Image& imageFloat, Eigen::Vector2f& brigthestPoint) +{ + image::Image patch; + std::array patchOrigin; + cutImage(imageFloat, newMask, patch, patchOrigin); + + image::Image convolutedPatch1; + image::Image convolutedPatch2; + + // Create Kernel + size_t kernelSize = round(patch.rows() / 40); // arbitrary + Eigen::VectorXf kernel(2 * kernelSize + 1); + createTriangleKernel(kernelSize, kernel); + + image::imageVerticalConvolution(patch, kernel, convolutedPatch1); + image::imageHorizontalConvolution(convolutedPatch1, kernel, convolutedPatch2); + + Eigen::Index maxRow, maxCol; + static_cast(convolutedPatch2.maxCoeff(&maxRow, &maxCol)); + + brigthestPoint(0) = maxCol + patchOrigin[0]; + brigthestPoint(1) = maxRow + patchOrigin[1]; } void createTriangleKernel(const size_t kernelSize, Eigen::VectorXf& kernel) @@ -240,6 +501,7 @@ void cutImage(const image::Image& imageFloat, image::Image& patch, std::array& patchOrigin) { + // Absolute position of the patch's top left corner in image const int minISphere = floor(sphereParam[1] - sphereParam[2] + imageFloat.rows() / 2); const int minJSphere = floor(sphereParam[0] - sphereParam[2] + imageFloat.cols() / 2); @@ -263,12 +525,50 @@ void cutImage(const image::Image& imageFloat, } } +void cutImage(const image::Image& imageFloat, + const image::Image& newMask, + image::Image& patch, + std::array& patchOrigin) +{ + int minISphere = newMask.rows(); + int minJSphere = newMask.cols(); + int maxISphere = 0; + int maxJSphere = 0; + + for (int j = 0; j < newMask.cols(); ++j) + { + for (int i = 0; i < newMask.rows(); ++i) + { + if (newMask(i, j) == 1) + { + if(minISphere > i) + minISphere = i; + + if(minJSphere > j) + minJSphere = j; + + if(maxISphere < i) + maxISphere = i; + + if(maxJSphere < j) + maxJSphere = j; + } + } + } + + patchOrigin[0] = minJSphere; + patchOrigin[1] = minISphere; + + patch = imageFloat.block(minISphere, minJSphere, maxISphere-minISphere, maxJSphere-minJSphere); +} + void writeJSON(const std::string& fileName, const sfmData::SfMData& sfmData, const std::vector& imageList, const Eigen::MatrixXf& lightMat, const std::vector& intList, - const bool saveAsModel) + const bool saveAsModel, + const std::string method) { bpt::ptree lightsTree; bpt::ptree fileTree; @@ -278,10 +578,19 @@ void writeJSON(const std::string& fileName, for (auto& viewIt : sfmData.getViews()) { std::map currentMetadata = sfmData.getView(viewIt.first).getImage().getMetadata(); - viewMap[currentMetadata.at("Exif:DateTimeDigitized")] = sfmData.getView(viewIt.first); + + if (currentMetadata.find("Exif:DateTimeDigitized") == currentMetadata.end()) + { + ALICEVISION_LOG_INFO("No metadata case (Exif:DateTimeDigitized is missing)"); + viewMap[sfmData.getView(viewIt.first).getImage().getImagePath()] = sfmData.getView(viewIt.first); + } + else + { + viewMap[currentMetadata.at("Exif:DateTimeDigitized")] = sfmData.getView(viewIt.first); + } } - for (const auto& [currentTime, viewId] : viewMap) + for (const auto& [currentId, viewId] : viewMap) { const fs::path imagePath = fs::path(viewId.getImage().getImagePath()); @@ -289,24 +598,26 @@ void writeJSON(const std::string& fileName, const bool calibratedFile = (std::find(imageList.begin(), imageList.end(), viewId.getImage().getImagePath()) != imageList.end()); // Only write images that were actually used for the lighting calibration, instead of all the input images - if (!boost::algorithm::icontains(imagePath.stem().string(), "ambiant") && calibratedFile) + if (!boost::algorithm::icontains(imagePath.stem().string(), "ambient") && calibratedFile) { bpt::ptree lightTree; if (saveAsModel) { lightTree.put("lightId", imgCpt); - lightTree.put("type", "directional"); } else { - // const IndexT index = viewId.getViewId lightTree.put("viewId", viewId.getViewId()); - lightTree.put("type", "directional"); } + if (!method.compare("SH")) + lightTree.put("type", "SH"); + else + lightTree.put("type", "directional"); // Light direction bpt::ptree directionNode; - for (int i = 0; i < 3; ++i) + int lightMatSize = lightMat.cols(); + for (int i = 0; i < lightMatSize; ++i) { bpt::ptree cell; cell.put_value(lightMat(imgCpt, i)); @@ -330,7 +641,7 @@ void writeJSON(const std::string& fileName, else { ALICEVISION_LOG_INFO("'" << imagePath << "' is in the input SfMData but has not been used for the lighting " - << "calibration or contains 'ambiant' in its filename."); + << "calibration or contains 'ambient' in its filename."); } } @@ -338,5 +649,52 @@ void writeJSON(const std::string& fileName, bpt::write_json(fileName, fileTree); } +void sphereFromLighting(const Eigen::VectorXf& lightVector, const float intensity, const std::string outputFileName, const int outputSize) +{ + float radius = (outputSize * 0.9) / 2; + image::Image pixelsValues(outputSize, outputSize); + + for (size_t j = 0; j < outputSize; ++j) + { + for (size_t i = 0; i < outputSize; ++i) + { + float center_xy = outputSize / 2; + Eigen::VectorXf normalSphere(lightVector.size()); + float distanceToCenter = sqrt((i - center_xy) * (i - center_xy) + (j - center_xy) * (j - center_xy)); + pixelsValues(i, j) = 0; + + if (distanceToCenter < radius) + { + normalSphere(0) = (float(j) - center_xy) / radius; + normalSphere(1) = (float(i) - center_xy) / radius; + normalSphere(2) = -sqrt(1 - normalSphere(0) * normalSphere(0) - normalSphere(1) * normalSphere(1)); + if (lightVector.size() > 3) + { + normalSphere(3) = 1; + } + if (lightVector.size() > 4) + { + normalSphere(4) = normalSphere(0) * normalSphere(1); + normalSphere(5) = normalSphere(0) * normalSphere(2); + normalSphere(6) = normalSphere(1) * normalSphere(2); + normalSphere(7) = normalSphere(0) * normalSphere(0) - normalSphere(1) * normalSphere(1); + normalSphere(8) = 3 * normalSphere(2) * normalSphere(2) - 1; + } + + for (size_t k = 0; k < lightVector.size(); ++k) + { + pixelsValues(i, j) += normalSphere(k) * lightVector(k); + } + pixelsValues(i, j) *= intensity; + } + } + } + + image::writeImage( + outputFileName, + pixelsValues, + image::ImageWriteOptions().toColorSpace(image::EImageColorSpace::NO_CONVERSION).storageDataType(image::EStorageDataType::Float)); +} + } // namespace lightingEstimation } // namespace aliceVision diff --git a/src/aliceVision/lightingEstimation/lightingCalibration.hpp b/src/aliceVision/lightingEstimation/lightingCalibration.hpp index 6dfd0a0044..5a24e76a2f 100644 --- a/src/aliceVision/lightingEstimation/lightingCalibration.hpp +++ b/src/aliceVision/lightingEstimation/lightingCalibration.hpp @@ -21,26 +21,41 @@ namespace lightingEstimation { * @param[in] sfmData Input .sfm file to calibrate from * @param[in] inputJSON Path to the JSON file containing the spheres parameters (see sphereDetection) * @param[out] outputPath Path to the JSON file in which lights' directions are written + * @param[in] method Method used for calibration ("brightestPoint" or "whiteSphere") + * @param[in] doDebug True to save debug images + * @param[in] saveAsModel True to save the estimated lights as model + * @param[in] ellipticEstimation True to use elliptic estimation of the lighting */ void lightCalibration(const sfmData::SfMData& sfmData, const std::string& inputJSON, const std::string& outputPath, const std::string& method, - const bool saveAsModel); + const bool doDebug, + const bool saveAsModel, + const bool ellipticEstimation); /** * @brief Calibrate lighting direction of an image containing a sphere * @param[in] picturePath Path to the image file * @param[in] sphereParam An array of 3 floating-point: the coordinates of the sphere center in the picture frame and the radius of the sphere * @param[in] focal Focal length of the camera - * @param[in] method Method used for calibration (only "brightestPoint" for now) + * @param[in] method Method used for calibration ("brightestPoint" or "whiteSphere") * @param[out] lightingDirection Output parameter for the estimated lighting direction */ void lightCalibrationOneImage(const std::string& picturePath, const std::array& sphereParam, const float focal, const std::string& method, - Eigen::Vector3f& lightingDirection); + Eigen::VectorXf& lightingDirection, + float& intensity); + +void calibrateLightFromRealSphere(const image::Image& imageFloat, + const cv::Mat& maskCV, + const Eigen::Matrix3f& K, + const float sphereRadius, + const std::string& method, + Eigen::VectorXf& lightingDirection, + float& intensity); /** * @brief Compute the brightest point on a sphere @@ -51,6 +66,8 @@ void lightCalibrationOneImage(const std::string& picturePath, */ void detectBrightestPoint(const std::array& sphereParam, const image::Image& imageFloat, Eigen::Vector2f& brigthestPoint); +void detectBrightestPoint(const image::Image newMask, const image::Image& imageFloat, Eigen::Vector2f& brigthestPoint); + /** * @brief Create a triangle kernel * @param[in] kernelSize Size of the kernel (should be an odd number) @@ -79,6 +96,11 @@ void cutImage(const image::Image& imageFloat, image::Image& patch, std::array& patchOrigin); +void cutImage(const image::Image& imageFloat, + const image::Image& newMask, + image::Image& patch, + std::array& patchOrigin); + /** * @brief Write a JSON file containing light information * @param[in] fileName The path to the JSON file to generate @@ -87,13 +109,17 @@ void cutImage(const image::Image& imageFloat, * @param[in] lightMat A matrix containing the directions of the light sources * @param[in] intList A vector of arrays containing the intensity of the light sources * @param[in] saveAsModel True to save the light IDs instead of the view IDs, false otherwise + * @param[in] method Name of the method for lighting estimation: whiteSphere, brightestPoint, SH */ void writeJSON(const std::string& fileName, const sfmData::SfMData& sfmData, const std::vector& imageList, const Eigen::MatrixXf& lightMat, const std::vector& intList, - const bool saveAsModel); + const bool saveAsModel, + const std::string method); + +void sphereFromLighting(const Eigen::VectorXf& lightVector, const float intensity, const std::string outputFileName, const int outputSize); } // namespace lightingEstimation } // namespace aliceVision diff --git a/src/aliceVision/lightingEstimation/lightingEstimation.cpp b/src/aliceVision/lightingEstimation/lightingEstimation.cpp index e0886bceec..359b9b38e1 100644 --- a/src/aliceVision/lightingEstimation/lightingEstimation.cpp +++ b/src/aliceVision/lightingEstimation/lightingEstimation.cpp @@ -32,7 +32,7 @@ void albedoNormalsProduct(MatrixXf& rhoTimesN, const MatrixXf& albedoChannel, co rhoTimesN(validIndex, 0) = albedoChannel(i) * augmentedNormals(i).nx(); rhoTimesN(validIndex, 1) = albedoChannel(i) * augmentedNormals(i).ny(); rhoTimesN(validIndex, 2) = albedoChannel(i) * augmentedNormals(i).nz(); - rhoTimesN(validIndex, 3) = albedoChannel(i) * augmentedNormals(i).nambiant(); + rhoTimesN(validIndex, 3) = albedoChannel(i) * augmentedNormals(i).nambient(); rhoTimesN(validIndex, 4) = albedoChannel(i) * augmentedNormals(i).nx_ny(); rhoTimesN(validIndex, 5) = albedoChannel(i) * augmentedNormals(i).nx_nz(); rhoTimesN(validIndex, 6) = albedoChannel(i) * augmentedNormals(i).ny_nz(); diff --git a/src/aliceVision/mesh/Texturing.cpp b/src/aliceVision/mesh/Texturing.cpp index 315674ff92..1e8bf40cd2 100644 --- a/src/aliceVision/mesh/Texturing.cpp +++ b/src/aliceVision/mesh/Texturing.cpp @@ -140,7 +140,7 @@ std::istream& operator>>(std::istream& in, EBumpMappingType& bumpMappingType) * coordinates of this pixel relative to \p triangle * @return */ -bool isPixelInTriangle(const Point2d* triangle, const Pixel& pixel, Point2d& barycentricCoords) +bool isPixelInTriangle(const Point2d* triangle, const Pixel& pixel, Point2d& barycentricCoords, double margin = 0.5) { // get pixel center GEO::vec2 p(pixel.x + 0.5, pixel.y + 0.5); @@ -154,7 +154,7 @@ bool isPixelInTriangle(const Point2d* triangle, const Pixel& pixel, Point2d& bar barycentricCoords.x = l3; barycentricCoords.y = l2; // tolerance threshold of 1/2 pixel for pixels on the edges of the triangle - return dist < 0.5 + std::numeric_limits::epsilon(); + return dist < margin + std::numeric_limits::epsilon(); } Point2d barycentricToCartesian(const Point2d* triangle, const Point2d& coords) @@ -167,6 +167,123 @@ Point3d barycentricToCartesian(const Point3d* triangle, const Point2d& coords) return triangle[0] + (triangle[2] - triangle[0]) * coords.x + (triangle[1] - triangle[0]) * coords.y; } +inline GEO::vec3 mesh_facet_interpolate_normal_at_point(const GEO::Mesh& mesh, GEO::index_t f, const GEO::vec3& p) +{ + const GEO::index_t v0 = mesh.facets.vertex(f, 0); + const GEO::index_t v1 = mesh.facets.vertex(f, 1); + const GEO::index_t v2 = mesh.facets.vertex(f, 2); + + const GEO::vec3 p0 = mesh.vertices.point(v0); + const GEO::vec3 p1 = mesh.vertices.point(v1); + const GEO::vec3 p2 = mesh.vertices.point(v2); + + const GEO::vec3 n0 = GEO::normalize(GEO::Geom::mesh_vertex_normal(mesh, v0)); + const GEO::vec3 n1 = GEO::normalize(GEO::Geom::mesh_vertex_normal(mesh, v1)); + const GEO::vec3 n2 = GEO::normalize(GEO::Geom::mesh_vertex_normal(mesh, v2)); + + GEO::vec3 barycCoords; + GEO::vec3 closestPoint; + GEO::Geom::point_triangle_squared_distance(p, p0, p1, p2, closestPoint, barycCoords.x, barycCoords.y, barycCoords.z); + + const GEO::vec3 n = barycCoords.x * n0 + barycCoords.y * n1 + barycCoords.z * n2; + + return GEO::normalize(n); +} + +inline GEO::vec3 mesh_facet_interpolate_normal_at_point(const StaticVector& ptsNormals, const Mesh& mesh, GEO::index_t f, const GEO::vec3& p) +{ + const GEO::index_t v0 = (mesh.tris)[f].v[0]; + const GEO::index_t v1 = (mesh.tris)[f].v[1]; + const GEO::index_t v2 = (mesh.tris)[f].v[2]; + + const GEO::vec3 p0((mesh.pts)[v0].x, (mesh.pts)[v0].y, (mesh.pts)[v0].z); + const GEO::vec3 p1((mesh.pts)[v1].x, (mesh.pts)[v1].y, (mesh.pts)[v1].z); + const GEO::vec3 p2((mesh.pts)[v2].x, (mesh.pts)[v2].y, (mesh.pts)[v2].z); + + const GEO::vec3 n0(ptsNormals[v0].x, ptsNormals[v0].y, ptsNormals[v0].z); + const GEO::vec3 n1(ptsNormals[v1].x, ptsNormals[v1].y, ptsNormals[v1].z); + const GEO::vec3 n2(ptsNormals[v2].x, ptsNormals[v2].y, ptsNormals[v2].z); + + GEO::vec3 barycCoords; + GEO::vec3 closestPoint; + GEO::Geom::point_triangle_squared_distance(p, p0, p1, p2, closestPoint, barycCoords.x, barycCoords.y, barycCoords.z); + + const GEO::vec3 n = barycCoords.x * n0 + barycCoords.y * n1 + barycCoords.z * n2; + + return GEO::normalize(n); +} + +template +inline Eigen::Matrix toEigen(const GEO::vecng& v) +{ + return Eigen::Matrix(v.data()); +} + +/** + * @brief Compute a transformation matrix to convert coordinates in world space coordinates into the triangle space. + * The triangle space is define by the Z-axis as the normal of the triangle, + * the X-axis aligned with the horizontal line in the texture file (using texture/UV coordinates). + * + * @param[in] mesh: input mesh + * @param[in] f: facet/triangle index + * @param[in] triPts: UV Coordinates + * @return Rotation matrix to convert from world space coordinates in the triangle space + */ +inline Eigen::Matrix3d computeTriangleTransform(const Mesh& mesh, int f, const Point2d* triPts) +{ + const Eigen::Vector3d p0 = toEigen((mesh.pts)[(mesh.tris)[f].v[0]]); + const Eigen::Vector3d p1 = toEigen((mesh.pts)[(mesh.tris)[f].v[1]]); + const Eigen::Vector3d p2 = toEigen((mesh.pts)[(mesh.tris)[f].v[2]]); + + const Eigen::Vector3d tX = (p1 - p0).normalized(); // edge0 => local triangle X-axis + const Eigen::Vector3d N = tX.cross((p2 - p0).normalized()).normalized(); // cross(edge0, edge1) => Z-axis + + // Correct triangle X-axis to be align with X-axis in the texture + const GEO::vec2 t0 = GEO::vec2(triPts[0].m); + const GEO::vec2 t1 = GEO::vec2(triPts[1].m); + const GEO::vec2 tV = GEO::normalize(t1 - t0); + const GEO::vec2 origNormal(1.0, 0.0); // X-axis in the texture + const double tAngle = GEO::Geom::angle(tV, origNormal); + Eigen::Matrix3d transform(Eigen::AngleAxisd(tAngle, N).toRotationMatrix()); + // Rotate triangle v0v1 axis around Z-axis, to get a X axis aligned with the 2d texture + Eigen::Vector3d X = (transform * tX).normalized(); + + const Eigen::Vector3d Y = N.cross(X).normalized(); // Y-axis + + Eigen::Matrix3d m; + m.col(0) = X; + m.col(1) = Y; + m.col(2) = N; + // const Eigen::Matrix3d mInv = m.inverse(); + const Eigen::Matrix3d mT = m.transpose(); + + return mT; +} + +inline void computeNormalHeight(const GEO::Mesh& mesh, + double orientation, + double t, + GEO::index_t f, + const Eigen::Matrix3d& m, + const GEO::vec3& q, + const GEO::vec3& qA, + const GEO::vec3& qB, + float& out_height, + image::RGBfColor& out_normal) +{ + GEO::vec3 intersectionPoint = t * qB + (1.0 - t) * qA; + out_height = q.distance(intersectionPoint) * orientation; + + // Use facet normal + // GEO::vec3 denseMeshNormal_f = normalize(GEO::Geom::mesh_facet_normal(mesh, f)); + // Use per pixel normal using weighted interpolation of the facet vertex normals + const GEO::vec3 denseMeshNormal = mesh_facet_interpolate_normal_at_point(mesh, f, intersectionPoint); + + Eigen::Vector3d dNormal = m * toEigen(denseMeshNormal); + dNormal.normalize(); + out_normal = image::RGBfColor(dNormal(0), dNormal(1), dNormal(2)); +} + void Texturing::generateUVsBasicMethod(mvsUtils::MultiViewParams& mp) { if (!mesh) @@ -282,7 +399,8 @@ void Texturing::updateAtlases() void Texturing::generateTextures(const mvsUtils::MultiViewParams& mp, const fs::path& outPath, size_t memoryAvailable, - image::EImageFileType textureFileType) + image::EImageFileType textureFileType, + mvsUtils::EFileType imageType) { // Ensure that contribution levels do not contain 0 and are sorted (as each frequency band contributes to lower bands). auto& m = texParams.multiBandNbContrib; @@ -363,7 +481,7 @@ void Texturing::generateTextures(const mvsUtils::MultiViewParams& mp, atlasIDs.push_back(atlasID); } ALICEVISION_LOG_INFO("Generating texture for atlases " << n * nbAtlasMax + 1 << " to " << n * nbAtlasMax + imax); - generateTexturesSubSet(mp, atlasIDs, imageCache, outPath, textureFileType); + generateTexturesSubSet(mp, atlasIDs, imageCache, outPath, textureFileType, imageType); } } @@ -371,7 +489,8 @@ void Texturing::generateTexturesSubSet(const mvsUtils::MultiViewParams& mp, const std::vector& atlasIDs, mvsUtils::ImagesCache>& imageCache, const fs::path& outPath, - image::EImageFileType textureFileType) + image::EImageFileType textureFileType, + mvsUtils::EFileType imageType) { if (atlasIDs.size() > _atlases.size()) throw std::runtime_error("Invalid atlas IDs "); @@ -740,7 +859,114 @@ void Texturing::generateTexturesSubSet(const mvsUtils::MultiViewParams& mp, } } } - writeTexture(atlasTexture, atlasID, outPath, textureFileType, -1); + + // If mode "normalMaps" + if (imageType == mvsUtils::EFileType::normalMap) + { + Eigen::Matrix visitedPixels(atlasTexture.img.rows(), atlasTexture.img.cols()); + visitedPixels.fill(false); + + // Rotation and normalization of normals +#pragma omp parallel for + for (int i = 0; i < static_cast(_atlases[atlasID].size()); ++i) + { + int triangleId = _atlases[atlasID][i]; + + // Retrieve triangle 3D and UV coordinates + Point2d triPixs[3]; + Point3d triPts[3]; + auto& triangleUvIds = mesh->trisUvIds[triangleId]; + + // Retrieve triangle normal + + // Compute the Bottom-Left minima of the current UDIM for [0,1] range remapping + Point2d udimBL; + StaticVector& uvCoords = mesh->uvCoords; + udimBL.x = std::floor(std::min({uvCoords[triangleUvIds[0]].x, uvCoords[triangleUvIds[1]].x, uvCoords[triangleUvIds[2]].x})); + udimBL.y = std::floor(std::min({uvCoords[triangleUvIds[0]].y, uvCoords[triangleUvIds[1]].y, uvCoords[triangleUvIds[2]].y})); + + for (int k = 0; k < 3; k++) + { + const int pointIndex = (mesh->tris)[triangleId].v[k]; + triPts[k] = (mesh->pts)[pointIndex]; // 3D coordinates + const int uvPointIndex = triangleUvIds.m[k]; + + Point2d uv = uvCoords[uvPointIndex]; + // UDIM: remap coordinates between [0,1] + uv = uv - udimBL; + + triPixs[k] = uv * texParams.textureSide; // UV coordinates + } + + // compute triangle bounding box in pixel indexes + // min values: floor(value) + // max values: ceil(value) + Pixel LU, RD; + LU.x = static_cast(std::floor(std::min({triPixs[0].x, triPixs[1].x, triPixs[2].x}))); + LU.y = static_cast(std::floor(std::min({triPixs[0].y, triPixs[1].y, triPixs[2].y}))); + RD.x = static_cast(std::ceil(std::max({triPixs[0].x, triPixs[1].x, triPixs[2].x}))); + RD.y = static_cast(std::ceil(std::max({triPixs[0].y, triPixs[1].y, triPixs[2].y}))); + + // sanity check: clamp values to [0; textureSide] + int texSide = static_cast(texParams.textureSide); + LU.x = clamp(LU.x, 0, texSide); + LU.y = clamp(LU.y, 0, texSide); + RD.x = clamp(RD.x, 0, texSide); + RD.y = clamp(RD.y, 0, texSide); + + const Eigen::Matrix3d worldToTriangleMatrix = computeTriangleTransform(*mesh, triangleId, triPixs); + + // iterate over bounding box's pixels + for (int y = LU.y; y < RD.y; ++y) + { + for (int x = LU.x; x < RD.x; ++x) + { + Pixel pix(x, y); // top-left corner of the pixel + Point2d barycCoords; + + // test if the pixel is inside triangle + // and retrieve its barycentric coordinates + const double margin = 0.5; + if (!isPixelInTriangle(triPixs, pix, barycCoords, margin)) + { + continue; + } + + // remap 'y' to image coordinates system (inverted Y axis) + const unsigned int y_ = (texParams.textureSide - 1) - y; + // 1D pixel index + const unsigned int xyoffset = y_ * texParams.textureSide + x; + + /* + // get 3D coordinates + const Point3d pt3d = barycentricToCartesian(triPts, barycCoords); + const GEO::vec3 q(pt3d.x, pt3d.y, pt3d.z); + + // Texel normal (weighted normal from the 3 vertices normals), instead of face normal for better + // transitions (reduce seams) + const GEO::vec3 triangleNormal_p = mesh_facet_interpolate_normal_at_point(sparseMesh, triangleId, q); + // const GEO::vec3 triangleNormal_p = GEO::vec3(triangleNormal.m); // to use the triangle normal instead + const GEO::vec3 scaledTriangleNormal = triangleNormal_p * minEdgeLength * 10; // ?????? + + */ + + Vec3 origNormal = atlasTexture.img(xyoffset).cast(); + if (visitedPixels(x, y)) + { + continue; + } + visitedPixels(x, y) = true; + + origNormal = worldToTriangleMatrix * origNormal; + origNormal.normalize(); + + origNormal = origNormal * 0.5 + Vec3(0.5, 0.5, 0.5); // Normal in visual representation + atlasTexture.img(xyoffset) = image::RGBfColor(origNormal[0], origNormal[1], origNormal[2]); + } + } + } + } + writeTexture(atlasTexture, atlasID, outPath, textureFileType, -1, imageType); } } @@ -768,7 +994,8 @@ void Texturing::writeTexture(AccuImage& atlasTexture, const std::size_t atlasID, const std::filesystem::path& outPath, image::EImageFileType textureFileType, - const int level) + const int level, + mvsUtils::EFileType imageType) { unsigned int outTextureSide = texParams.textureSide; // WARNING: we modify the "imgCount" to apply the padding (to avoid the creation of a new buffer) @@ -891,14 +1118,34 @@ void Texturing::writeTexture(AccuImage& atlasTexture, ALICEVISION_LOG_INFO(" - Downscaling texture (" << texParams.downscale << "x)."); imageAlgo::resizeImage(texParams.downscale, atlasTexture.img, resizedColorBuffer); + + // Normalize normal map after dowscaling + /*if(imageType == mvsUtils::EFileType::normalMap) + { + for(int i = 0; i < resizedColorBuffer.size(); ++i) + { + resizedColorBuffer(i).normalize(); + } + }*/ + std::swap(resizedColorBuffer, atlasTexture.img); } - material.diffuseType = textureFileType; - const std::string textureName = material.textureName(Material::TextureType::DIFFUSE, static_cast(atlasID)); - material.addTexture(Material::TextureType::DIFFUSE, textureName); + std::string textureName; + if (imageType == mvsUtils::EFileType::normalMap) + { + material.bumpType = textureFileType; + textureName = material.textureName(Material::TextureType::BUMP, static_cast(atlasID)); + material.addTexture(Material::TextureType::BUMP, textureName); + } + else + { + material.diffuseType = textureFileType; + textureName = material.textureName(Material::TextureType::DIFFUSE, static_cast(atlasID)); + material.addTexture(Material::TextureType::DIFFUSE, textureName); + } - fs::path texturePath = outPath / textureName; + const fs::path texturePath = outPath / textureName; ALICEVISION_LOG_INFO(" - Writing texture file: " << texturePath.string()); image::writeImage(texturePath.string(), @@ -1206,123 +1453,6 @@ void Texturing::saveAs(const fs::path& dir, const std::string& basename, EFileTy ALICEVISION_LOG_INFO("Save mesh to " << meshFileTypeStr << " done."); } -inline GEO::vec3 mesh_facet_interpolate_normal_at_point(const GEO::Mesh& mesh, GEO::index_t f, const GEO::vec3& p) -{ - const GEO::index_t v0 = mesh.facets.vertex(f, 0); - const GEO::index_t v1 = mesh.facets.vertex(f, 1); - const GEO::index_t v2 = mesh.facets.vertex(f, 2); - - const GEO::vec3 p0 = mesh.vertices.point(v0); - const GEO::vec3 p1 = mesh.vertices.point(v1); - const GEO::vec3 p2 = mesh.vertices.point(v2); - - const GEO::vec3 n0 = GEO::normalize(GEO::Geom::mesh_vertex_normal(mesh, v0)); - const GEO::vec3 n1 = GEO::normalize(GEO::Geom::mesh_vertex_normal(mesh, v1)); - const GEO::vec3 n2 = GEO::normalize(GEO::Geom::mesh_vertex_normal(mesh, v2)); - - GEO::vec3 barycCoords; - GEO::vec3 closestPoint; - GEO::Geom::point_triangle_squared_distance(p, p0, p1, p2, closestPoint, barycCoords.x, barycCoords.y, barycCoords.z); - - const GEO::vec3 n = barycCoords.x * n0 + barycCoords.y * n1 + barycCoords.z * n2; - - return GEO::normalize(n); -} - -inline GEO::vec3 mesh_facet_interpolate_normal_at_point(const StaticVector& ptsNormals, const Mesh& mesh, GEO::index_t f, const GEO::vec3& p) -{ - const GEO::index_t v0 = (mesh.tris)[f].v[0]; - const GEO::index_t v1 = (mesh.tris)[f].v[1]; - const GEO::index_t v2 = (mesh.tris)[f].v[2]; - - const GEO::vec3 p0((mesh.pts)[v0].x, (mesh.pts)[v0].y, (mesh.pts)[v0].z); - const GEO::vec3 p1((mesh.pts)[v1].x, (mesh.pts)[v1].y, (mesh.pts)[v1].z); - const GEO::vec3 p2((mesh.pts)[v2].x, (mesh.pts)[v2].y, (mesh.pts)[v2].z); - - const GEO::vec3 n0(ptsNormals[v0].x, ptsNormals[v0].y, ptsNormals[v0].z); - const GEO::vec3 n1(ptsNormals[v1].x, ptsNormals[v1].y, ptsNormals[v1].z); - const GEO::vec3 n2(ptsNormals[v2].x, ptsNormals[v2].y, ptsNormals[v2].z); - - GEO::vec3 barycCoords; - GEO::vec3 closestPoint; - GEO::Geom::point_triangle_squared_distance(p, p0, p1, p2, closestPoint, barycCoords.x, barycCoords.y, barycCoords.z); - - const GEO::vec3 n = barycCoords.x * n0 + barycCoords.y * n1 + barycCoords.z * n2; - - return GEO::normalize(n); -} - -template -inline Eigen::Matrix toEigen(const GEO::vecng& v) -{ - return Eigen::Matrix(v.data()); -} - -/** - * @brief Compute a transformation matrix to convert coordinates in world space coordinates into the triangle space. - * The triangle space is define by the Z-axis as the normal of the triangle, - * the X-axis aligned with the horizontal line in the texture file (using texture/UV coordinates). - * - * @param[in] mesh: input mesh - * @param[in] f: facet/triangle index - * @param[in] triPts: UV Coordinates - * @return Rotation matrix to convert from world space coordinates in the triangle space - */ -inline Eigen::Matrix3d computeTriangleTransform(const Mesh& mesh, int f, const Point2d* triPts) -{ - const Eigen::Vector3d p0 = toEigen((mesh.pts)[(mesh.tris)[f].v[0]]); - const Eigen::Vector3d p1 = toEigen((mesh.pts)[(mesh.tris)[f].v[1]]); - const Eigen::Vector3d p2 = toEigen((mesh.pts)[(mesh.tris)[f].v[2]]); - - const Eigen::Vector3d tX = (p1 - p0).normalized(); // edge0 => local triangle X-axis - const Eigen::Vector3d N = tX.cross((p2 - p0).normalized()).normalized(); // cross(edge0, edge1) => Z-axis - - // Correct triangle X-axis to be align with X-axis in the texture - const GEO::vec2 t0 = GEO::vec2(triPts[0].m); - const GEO::vec2 t1 = GEO::vec2(triPts[1].m); - const GEO::vec2 tV = GEO::normalize(t1 - t0); - const GEO::vec2 origNormal(1.0, 0.0); // X-axis in the texture - const double tAngle = GEO::Geom::angle(tV, origNormal); - Eigen::Matrix3d transform(Eigen::AngleAxisd(tAngle, N).toRotationMatrix()); - // Rotate triangle v0v1 axis around Z-axis, to get a X axis aligned with the 2d texture - Eigen::Vector3d X = (transform * tX).normalized(); - - const Eigen::Vector3d Y = N.cross(X).normalized(); // Y-axis - - Eigen::Matrix3d m; - m.col(0) = X; - m.col(1) = Y; - m.col(2) = N; - // const Eigen::Matrix3d mInv = m.inverse(); - const Eigen::Matrix3d mT = m.transpose(); - - return mT; -} - -inline void computeNormalHeight(const GEO::Mesh& mesh, - double orientation, - double t, - GEO::index_t f, - const Eigen::Matrix3d& m, - const GEO::vec3& q, - const GEO::vec3& qA, - const GEO::vec3& qB, - float& out_height, - image::RGBfColor& out_normal) -{ - GEO::vec3 intersectionPoint = t * qB + (1.0 - t) * qA; - out_height = q.distance(intersectionPoint) * orientation; - - // Use facet normal - // GEO::vec3 denseMeshNormal_f = normalize(GEO::Geom::mesh_facet_normal(mesh, f)); - // Use per pixel normal using weighted interpolation of the facet vertex normals - const GEO::vec3 denseMeshNormal = mesh_facet_interpolate_normal_at_point(mesh, f, intersectionPoint); - - Eigen::Vector3d dNormal = m * toEigen(denseMeshNormal); - dNormal.normalize(); - out_normal = image::RGBfColor(dNormal(0), dNormal(1), dNormal(2)); -} - void Texturing::_generateNormalAndHeightMaps(const mvsUtils::MultiViewParams& mp, const GEO::MeshFacetsAABB& denseMeshAABB, const GEO::Mesh& sparseMesh, diff --git a/src/aliceVision/mesh/Texturing.hpp b/src/aliceVision/mesh/Texturing.hpp index 601b8eb111..545ad0913b 100644 --- a/src/aliceVision/mesh/Texturing.hpp +++ b/src/aliceVision/mesh/Texturing.hpp @@ -196,14 +196,16 @@ struct Texturing void generateTextures(const mvsUtils::MultiViewParams& mp, const fs::path& outPath, size_t memoryAvailable, - image::EImageFileType textureFileType = image::EImageFileType::PNG); + image::EImageFileType textureFileType = image::EImageFileType::PNG, + mvsUtils::EFileType imageType = mvsUtils::EFileType::none); /// Generate texture files for the given sub-set of texture atlases void generateTexturesSubSet(const mvsUtils::MultiViewParams& mp, const std::vector& atlasIDs, mvsUtils::ImagesCache>& imageCache, const fs::path& outPath, - image::EImageFileType textureFileType = image::EImageFileType::PNG); + image::EImageFileType textureFileType = image::EImageFileType::PNG, + mvsUtils::EFileType imageType = mvsUtils::EFileType::none); void generateNormalAndHeightMaps(const mvsUtils::MultiViewParams& mp, const Mesh& denseMesh, @@ -223,7 +225,8 @@ struct Texturing const std::size_t atlasID, const fs::path& outPath, image::EImageFileType textureFileType, - const int level); + const int level, + mvsUtils::EFileType imageType = mvsUtils::EFileType::none); /// Save textured mesh as an OBJ + MTL file void saveAs(const fs::path& dir, const std::string& basename, aliceVision::mesh::EFileType meshFileType = aliceVision::mesh::EFileType::OBJ); diff --git a/src/aliceVision/mvsUtils/MultiViewParams.cpp b/src/aliceVision/mvsUtils/MultiViewParams.cpp index 98cf1d4557..a1e7f6540b 100644 --- a/src/aliceVision/mvsUtils/MultiViewParams.cpp +++ b/src/aliceVision/mvsUtils/MultiViewParams.cpp @@ -38,7 +38,7 @@ MultiViewParams::MultiViewParams(const sfmData::SfMData& sfmData, const std::string& imagesFolder, const std::string& depthMapsFolder, const std::string& depthMapsFilterFolder, - bool readFromDepthMaps, + mvsUtils::EFileType fileType, int downscale) : _sfmData(sfmData), _imagesFolder(imagesFolder + "/"), @@ -63,7 +63,7 @@ MultiViewParams::MultiViewParams(const sfmData::SfMData& sfmData, std::string path = view.getImage().getImagePath(); - if (readFromDepthMaps) + if (fileType == mvsUtils::EFileType::depthMap) { if (depthMapsFolder.empty()) { @@ -76,6 +76,10 @@ MultiViewParams::MultiViewParams(const sfmData::SfMData& sfmData, path = getFileNameFromViewId(*this, view.getViewId(), mvsUtils::EFileType::depthMap); } } + else if (fileType == mvsUtils::EFileType::normalMap) + { + path = getFileNameFromViewId(*this, view.getViewId(), mvsUtils::EFileType::normalMap); + } else if (_imagesFolder != "/" && !_imagesFolder.empty() && fs::is_directory(_imagesFolder) && !fs::is_empty(_imagesFolder)) { // find folder file extension diff --git a/src/aliceVision/mvsUtils/MultiViewParams.hpp b/src/aliceVision/mvsUtils/MultiViewParams.hpp index 404ed430eb..04648a0331 100644 --- a/src/aliceVision/mvsUtils/MultiViewParams.hpp +++ b/src/aliceVision/mvsUtils/MultiViewParams.hpp @@ -80,6 +80,7 @@ enum class EFileType volumeTopographicCut = 50, stats9p = 51, tilePattern = 52, + none = 9999 }; class MultiViewParams @@ -116,7 +117,7 @@ class MultiViewParams const std::string& imagesFolder = "", const std::string& depthMapsFolder = "", const std::string& depthMapsFilterFolder = "", - bool readFromDepthMaps = false, + mvsUtils::EFileType fileType = mvsUtils::EFileType::none, int downscale = 1); ~MultiViewParams(); diff --git a/src/aliceVision/mvsUtils/fileIO.cpp b/src/aliceVision/mvsUtils/fileIO.cpp index c641ed12b8..8223310c9a 100644 --- a/src/aliceVision/mvsUtils/fileIO.cpp +++ b/src/aliceVision/mvsUtils/fileIO.cpp @@ -336,6 +336,10 @@ std::string getFileNameFromViewId(const MultiViewParams& mp, ext = "obj"; break; } + case EFileType::none: + { + ALICEVISION_THROW_ERROR("FileType is None"); + } } const std::string fileName = folder + std::to_string(viewId) + suffix + customSuffix + tileSuffix + "." + ext; diff --git a/src/aliceVision/photometricStereo/normalIntegration.cpp b/src/aliceVision/photometricStereo/normalIntegration.cpp index b6ae41250a..f14f01e7b3 100644 --- a/src/aliceVision/photometricStereo/normalIntegration.cpp +++ b/src/aliceVision/photometricStereo/normalIntegration.cpp @@ -348,7 +348,7 @@ void DCTIntegration(const image::Image& normals, { double denom = 4 * (pow(sin(0.5 * M_PI * j / nbCols), 2) + pow(sin(0.5 * M_PI * i / nbRows), 2)); denom = std::max(denom, 0.0001); - z_bar_bar.at(i, j) = fcos.at(i, j) / denom; + z_bar_bar.at(i, j) = -fcos.at(i, j) / denom; } } diff --git a/src/aliceVision/photometricStereo/photometricDataIO.cpp b/src/aliceVision/photometricStereo/photometricDataIO.cpp index 174337eb35..77c0dfe400 100644 --- a/src/aliceVision/photometricStereo/photometricDataIO.cpp +++ b/src/aliceVision/photometricStereo/photometricDataIO.cpp @@ -94,11 +94,11 @@ void loadLightDirections(const std::string& dirFileName, const Eigen::MatrixXf& } } -void loadLightHS(const std::string& dirFileName, Eigen::MatrixXf& lightMat) +void loadLightSH(const std::string& dirFileName, Eigen::MatrixXf& lightMat) { std::stringstream stream; std::string line; - float x, y, z, ambiant, nxny, nxnz, nynz, nx2ny2, nz2; + float x, y, z, ambient, nxny, nxnz, nynz, nx2ny2, nz2; std::fstream dirFile; dirFile.open(dirFileName, std::ios::in); @@ -118,13 +118,13 @@ void loadLightHS(const std::string& dirFileName, Eigen::MatrixXf& lightMat) stream.clear(); stream.str(line); - stream >> x >> y >> z >> ambiant >> nxny >> nxnz >> nynz >> nx2ny2 >> nz2; + stream >> x >> y >> z >> ambient >> nxny >> nxnz >> nynz >> nx2ny2 >> nz2; if (lineNumber < lightMat.rows()) { lightMat(lineNumber, 0) = x; lightMat(lineNumber, 1) = -y; lightMat(lineNumber, 2) = -z; - lightMat(lineNumber, 3) = ambiant; + lightMat(lineNumber, 3) = ambient; lightMat(lineNumber, 4) = nxny; lightMat(lineNumber, 5) = nxnz; lightMat(lineNumber, 6) = nynz; @@ -195,6 +195,9 @@ void buildLightMatFromJSON(const std::string& fileName, int cpt = 0; for (auto& light : fileTree.get_child("lights")) { + if (lineNumber == lightMat.rows()) + break; + IndexT lightIndex = light.second.get("lightId", UndefinedIndexT); if (lightIndex != UndefinedIndexT) { @@ -205,14 +208,21 @@ void buildLightMatFromJSON(const std::string& fileName, ++cpt; } intList.push_back(currentIntensities); - cpt = 0; - for (auto& direction : light.second.get_child("direction")) { - lightMat(lightIndex, cpt) = direction.second.get_value(); + if (cpt < 3) + { + lightMat(lineNumber, cpt) = direction.second.get_value(); + } + else if (cpt == 4) + { + lightMat.row(lineNumber) = lightMat.row(lineNumber)/lightMat.row(lineNumber).norm(); + ALICEVISION_LOG_INFO("SH will soon be available for use in PS. For now, lighting is reduced to directional"); + } ++cpt; } + ++lineNumber; } else { @@ -231,16 +241,79 @@ void buildLightMatFromJSON(const std::string& fileName, for (auto& direction : light.second.get_child("direction")) { - lightMat(lineNumber, cpt) = direction.second.get_value(); + if (cpt < 3) + { + lightMat(lineNumber, cpt) = direction.second.get_value(); + } + else if (cpt == 4) + { + // no support for SH lighting : + lightMat.row(lineNumber) = lightMat.row(lineNumber)/lightMat.row(lineNumber).norm(); + ALICEVISION_LOG_INFO("SH will soon be available for use in PS. For now, lighting is reduced to directional"); + } ++cpt; } ++lineNumber; + break; } } } } } +void buildLightMatFromLP(const std::string& fileName, + const std::vector& imageList, + Eigen::MatrixXf& lightMat, + std::vector>& intList) +{ + std::string pictureName; + float x, y, z; + + int lineNumber = 0; + std::array intensities = {1.0, 1.0, 1.0}; + + for (auto& currentImPath : imageList) + { + std::stringstream stream; + std::string line; + std::fstream intFile; + intFile.open(fileName, std::ios::in); + + if (!intFile.is_open()) + { + ALICEVISION_LOG_ERROR("Unable to load Lp file"); + ALICEVISION_THROW_ERROR("Cannot open '" << fileName << "'!"); + } + else + { + fs::path imagePathFS = fs::path(currentImPath); + while (!intFile.eof()) + { + std::getline(intFile, line); + stream.clear(); + stream.str(line); + + stream >> pictureName >> x >> y >> z; + + std::string stringToCompare = imagePathFS.filename().string(); + + if (boost::algorithm::iequals(pictureName, stringToCompare)) + { + lightMat(lineNumber, 0) = x; + lightMat(lineNumber, 1) = -y; + lightMat(lineNumber, 2) = -z; + ++lineNumber; + + intList.push_back(intensities); + + break; + } + } + intFile.close(); + } + } +} + void loadMask(std::string const& maskName, image::Image& mask) { if (maskName.empty() || !utils::exists(maskName)) @@ -281,6 +354,29 @@ void getIndMask(image::Image const& mask, std::vector& indices) } } +void getIndMask(image::Image const& mask, std::vector& indices, image::Image& indexInMask) +{ + const int nbRows = mask.rows(); + const int nbCols = mask.cols(); + + for (int j = 0; j < nbCols; ++j) + { + for (int i = 0; i < nbRows; ++i) + { + if (mask(i, j) > 0.7) + { + int currentIndex = j * nbRows + i; + indices.push_back(currentIndex); + indexInMask(i, j) = indices.size() - 1; + } + else + { + indexInMask(i, j) = -1; + } + } + } +} + void intensityScaling(std::array const& intensities, image::Image& imageToScale) { int nbRows = imageToScale.rows(); @@ -396,9 +492,9 @@ void convertNormalMap2png(const image::Image& normalsIm, image } else { - normalsImPNG(i, j)(0) = floor(255 * (normalsIm(i, j)(0) + 1) / 2); - normalsImPNG(i, j)(1) = -floor(255 * (normalsIm(i, j)(1) + 1) / 2); - normalsImPNG(i, j)(2) = -floor(255 * normalsIm(i, j)(2)); + normalsImPNG(i, j)(0) = floor(255.0 * (normalsIm(i, j)(0) + 1.0) / 2.0); + normalsImPNG(i, j)(1) = -ceil(255.0 * (normalsIm(i, j)(1) + 1.0) / 2.0); + normalsImPNG(i, j)(2) = -ceil(255.0 * normalsIm(i, j)(2)); } } } @@ -433,10 +529,18 @@ void writePSResults(const std::string& outputPath, const image::Image normalsImPNG(normals.cols(), normals.rows()); + convertNormalMap2png(normals, normalsImPNG); image::writeImage( - outputPath + "/albedo.exr", - albedo, + outputPath + "/normals.png", + normalsImPNG, image::ImageWriteOptions().toColorSpace(image::EImageColorSpace::NO_CONVERSION).storageDataType(image::EStorageDataType::Float)); + + image::writeImage( + outputPath + "/albedo.png", + albedo, + image::ImageWriteOptions().toColorSpace(image::EImageColorSpace::NO_CONVERSION).storageDataType(image::EStorageDataType::Half)); } void writePSResults(const std::string& outputPath, @@ -448,10 +552,23 @@ void writePSResults(const std::string& outputPath, outputPath + "/" + std::to_string(poseId) + "_normals.exr", normals, image::ImageWriteOptions().toColorSpace(image::EImageColorSpace::NO_CONVERSION).storageDataType(image::EStorageDataType::Float)); + + image::Image normalsImPNG(normals.cols(), normals.rows()); + convertNormalMap2png(normals, normalsImPNG); + image::writeImage( + outputPath + "/" + std::to_string(poseId) + "_normals.png", + normalsImPNG, + image::ImageWriteOptions().toColorSpace(image::EImageColorSpace::NO_CONVERSION).storageDataType(image::EStorageDataType::Float)); + image::writeImage( outputPath + "/" + std::to_string(poseId) + "_albedo.exr", albedo, image::ImageWriteOptions().toColorSpace(image::EImageColorSpace::NO_CONVERSION).storageDataType(image::EStorageDataType::Float)); + + image::writeImage( + outputPath + "/" + std::to_string(poseId) + "_albedo.png", + albedo, + image::ImageWriteOptions().toColorSpace(image::EImageColorSpace::NO_CONVERSION).storageDataType(image::EStorageDataType::Half)); } } // namespace photometricStereo diff --git a/src/aliceVision/photometricStereo/photometricDataIO.hpp b/src/aliceVision/photometricStereo/photometricDataIO.hpp index e1e71bc4fc..18d9e39467 100644 --- a/src/aliceVision/photometricStereo/photometricDataIO.hpp +++ b/src/aliceVision/photometricStereo/photometricDataIO.hpp @@ -40,7 +40,7 @@ void loadLightDirections(const std::string& dirFileName, const Eigen::MatrixXf& * @param[in] dirFileName Path to the direction file * @param[out] lightMat Matrix of directions of light */ -void loadLightHS(const std::string& dirFileName, Eigen::MatrixXf& lightMat); +void loadLightSH(const std::string& dirFileName, Eigen::MatrixXf& lightMat); /** * @brief Load light data from a JSON file to an Eigen matrix (with a list of images) @@ -66,6 +66,11 @@ void buildLightMatFromJSON(const std::string& fileName, Eigen::MatrixXf& lightMat, std::vector>& intList); +void buildLightMatFromLP(const std::string& fileName, + const std::vector& imageList, + Eigen::MatrixXf& lightMat, + std::vector>& intList); + /** * @brief Load a mask * @param[in] maskName Path to mask @@ -80,6 +85,8 @@ void loadMask(std::string const& maskName, image::Image& mask); */ void getIndMask(image::Image const& mask, std::vector& indices); +void getIndMask(image::Image const& mask, std::vector& indices, image::Image& indexInMask); + /** * @brief Apply the intensities to each channel of each image * @param[in] intensities Intensity values to apply to the image diff --git a/src/aliceVision/photometricStereo/photometricStereo.cpp b/src/aliceVision/photometricStereo/photometricStereo.cpp index 59b4b82b4f..2a3cc1c7a6 100644 --- a/src/aliceVision/photometricStereo/photometricStereo.cpp +++ b/src/aliceVision/photometricStereo/photometricStereo.cpp @@ -35,9 +35,9 @@ void photometricStereo(const std::string& inputPath, image::Image& albedo) { size_t dim = 3; - if (PSParameters.SHOrder == 2) + if (PSParameters.SHOrder != 0) { - dim = 9; + ALICEVISION_LOG_INFO("SH will soon be available for use in PS. For now, lighting is reduced to directional"); } std::vector imageList; @@ -61,21 +61,21 @@ void photometricStereo(const std::string& inputPath, std::string maskName = lightDataPath.remove_filename().string() + "/mask.png"; loadMask(maskName, mask); - std::string pathToAmbiant = ""; + std::string pathToAmbient = ""; - if (PSParameters.removeAmbiant) + if (PSParameters.removeAmbient) { for (auto& currentPath : imageList) { const fs::path imagePath = fs::path(currentPath); - if (boost::algorithm::icontains(imagePath.stem().string(), "ambiant")) + if (boost::algorithm::icontains(imagePath.stem().string(), "ambient")) { - pathToAmbiant = imagePath.string(); + pathToAmbient = imagePath.string(); } } } - photometricStereo(imageList, intList, lightMat, mask, pathToAmbiant, PSParameters, normals, albedo); + photometricStereo(imageList, intList, lightMat, mask, pathToAmbient, PSParameters, normals, albedo); writePSResults(outputPath, normals, albedo); image::writeImage(outputPath + "/mask.png", mask, image::ImageWriteOptions().toColorSpace(image::EImageColorSpace::NO_CONVERSION)); @@ -92,12 +92,12 @@ void photometricStereo(const sfmData::SfMData& sfmData, bool skipAll = true; bool groupedImages = false; size_t dim = 3; - if (PSParameters.SHOrder == 2) + if (PSParameters.SHOrder != 0) { - dim = 9; + ALICEVISION_LOG_INFO("SH will soon be available for use in PS. For now, lighting is reduced to directional"); } - std::string pathToAmbiant = ""; + std::string pathToAmbient = ""; std::map> viewsPerPoseId; for (auto& viewIt : sfmData.getViews()) @@ -112,15 +112,31 @@ void photometricStereo(const sfmData::SfMData& sfmData, ALICEVISION_LOG_INFO("Pose Id: " << posesIt.first); std::vector& initViewIds = posesIt.second; + bool hasMetadata = true; + std::map metadataTest = sfmData.getView(initViewIds.at(0)).getImage().getMetadata(); + if (metadataTest.find("Exif:DateTimeDigitized") == metadataTest.end()) + hasMetadata = false; + + std::vector viewIds; std::map idMap; - for (auto& viewId : initViewIds) + + if (hasMetadata) { - std::map currentMetadata = sfmData.getView(viewId).getImage().getMetadata(); - idMap[currentMetadata.at("Exif:DateTimeDigitized")] = viewId; + for (auto& viewId : initViewIds) + { + std::map currentMetadata = sfmData.getView(viewId).getImage().getMetadata(); + idMap[currentMetadata.at("Exif:DateTimeDigitized")] = viewId; + } } - - std::vector viewIds; - for (const auto& [currentTime, viewId] : idMap) + else + { + for (auto& viewId : initViewIds) + { + const fs::path imagePath = fs::path(sfmData.getView(viewId).getImage().getImagePath()); + idMap[imagePath.string()] = viewId; + } + } + for (const auto& [currentId, viewId] : idMap) { viewIds.push_back(viewId); } @@ -128,14 +144,15 @@ void photometricStereo(const sfmData::SfMData& sfmData, for (auto& viewId : viewIds) { const fs::path imagePath = fs::path(sfmData.getView(viewId).getImage().getImagePath()); - if (!boost::algorithm::icontains(imagePath.stem().string(), "ambiant")) + if (!boost::algorithm::icontains(imagePath.stem().string(), "ambient")) { ALICEVISION_LOG_INFO(" - " << imagePath.string()); imageList.push_back(imagePath.string()); } - else if (PSParameters.removeAmbiant) + else if (PSParameters.removeAmbient) { - pathToAmbiant = imagePath.string(); + ALICEVISION_LOG_INFO("Remove ambient light - " << imagePath.string()); + pathToAmbient = imagePath.string(); } } @@ -148,7 +165,16 @@ void photometricStereo(const sfmData::SfMData& sfmData, } else { - buildLightMatFromJSON(lightData, viewIds, lightMat, intList); + const std::string extension = fs::path(lightData).extension().string(); + + if (extension == ".json") // JSON File + { + buildLightMatFromJSON(lightData, viewIds, lightMat, intList); + } + else if (extension == ".lp") + { + buildLightMatFromLP(lightData, imageList, lightMat, intList); + } } /* Ensure that there are as many images as light calibrations, and that the list of input images is not empty. @@ -180,7 +206,7 @@ void photometricStereo(const sfmData::SfMData& sfmData, loadMask(currentMaskPath, mask); - photometricStereo(imageList, intList, lightMat, mask, pathToAmbiant, PSParameters, normals, albedo); + photometricStereo(imageList, intList, lightMat, mask, pathToAmbient, PSParameters, normals, albedo); writePSResults(outputPath, normals, albedo, posesIt.first); image::writeImage(outputPath + "/" + std::to_string(posesIt.first) + "_mask.png", @@ -197,6 +223,14 @@ void photometricStereo(const sfmData::SfMData& sfmData, outputPath + "/" + std::to_string(posesIt.first) + "_normals_w.exr", normals, image::ImageWriteOptions().toColorSpace(image::EImageColorSpace::NO_CONVERSION).storageDataType(image::EStorageDataType::Float)); + + + image::Image normalsImPNG(normals.cols(), normals.rows()); + convertNormalMap2png(normals, normalsImPNG); + image::writeImage( + outputPath + "/" + std::to_string(posesIt.first) + "_normals_w.png", + normalsImPNG, + image::ImageWriteOptions().toColorSpace(image::EImageColorSpace::NO_CONVERSION).storageDataType(image::EStorageDataType::Float)); } if (skipAll) @@ -253,13 +287,28 @@ void photometricStereo(const sfmData::SfMData& sfmData, sfmDataIO::save( normalSfmData, outputPath + "/normalMaps.sfm", sfmDataIO::ESfMData(sfmDataIO::VIEWS | sfmDataIO::INTRINSICS | sfmDataIO::EXTRINSICS)); + + // Create Normal_png SfmData + sfmData::SfMData normalSfmDataPNG = albedoSfmData; + for (auto& viewIt : normalSfmDataPNG.getViews()) + { + const IndexT viewId = viewIt.first; + IndexT poseId = viewIt.second->getPoseId(); + + sfmData::View* view = normalSfmDataPNG.getViews().at(viewId).get(); + std::string imagePath = outputPath + "/" + std::to_string(poseId) + "_normals_w.png"; + view->getImage().setImagePath(imagePath); + } + + sfmDataIO::save( + normalSfmDataPNG, outputPath + "/normalMapsPNG.sfm", sfmDataIO::ESfMData(sfmDataIO::VIEWS | sfmDataIO::INTRINSICS | sfmDataIO::EXTRINSICS)); } void photometricStereo(const std::vector& imageList, const std::vector>& intList, const Eigen::MatrixXf& lightMat, image::Image& mask, - const std::string& pathToAmbiant, + const std::string& pathToAmbient, const PhotometricSteroParameters& PSParameters, image::Image& normals, image::Image& albedo) @@ -306,37 +355,41 @@ void photometricStereo(const std::vector& imageList, for (int i = 0; i < maskSize; ++i) indices.push_back(i); + + hasMask = true; } - // Read ambiant - image::Image imageAmbiant; + // Read ambient + image::Image imageAmbient; - if (boost::algorithm::icontains(fs::path(pathToAmbiant).stem().string(), "ambiant")) + if (boost::algorithm::icontains(fs::path(pathToAmbient).stem().string(), "ambient")) { - ALICEVISION_LOG_INFO("Removing ambiant light"); - ALICEVISION_LOG_INFO(pathToAmbiant); + ALICEVISION_LOG_INFO("Removing ambient light"); + ALICEVISION_LOG_INFO(pathToAmbient); - image::readImage(pathToAmbiant, imageAmbiant, image::EImageColorSpace::NO_CONVERSION); + image::readImage(pathToAmbient, imageAmbient, image::EImageColorSpace::NO_CONVERSION); if (PSParameters.downscale > 1) { - imageAlgo::resizeImage(PSParameters.downscale, imageAmbiant); + imageAlgo::resizeImage(PSParameters.downscale, imageAmbient); } } // Tiling int auxMaskSize = maskSize; - int numberOfPixels = auxMaskSize * imageList.size() * 3; + int numberOfPixels = auxMaskSize * 3; + int numberOfMasks = 1; - while (numberOfPixels > sizeMax) + while (numberOfPixels > (sizeMax / imageList.size())) { numberOfMasks = numberOfMasks * 2; - auxMaskSize = floor(auxMaskSize / 4); - numberOfPixels = auxMaskSize * imageList.size() * 3; + auxMaskSize = floor(auxMaskSize / 2); + numberOfPixels = floor(numberOfPixels / 2); + hasMask = true; } - Eigen::MatrixXf normalsVect = Eigen::MatrixXf::Zero(lightMat.cols(), pictRows * pictCols); + Eigen::MatrixXf normalsVect = Eigen::MatrixXf::Zero(3, pictRows * pictCols); Eigen::MatrixXf albedoVect = Eigen::MatrixXf::Zero(3, pictRows * pictCols); int remainingPixels = maskSize; @@ -384,9 +437,9 @@ void photometricStereo(const std::vector& imageList, imageAlgo::resizeImage(PSParameters.downscale, imageFloat); } - if (boost::algorithm::icontains(fs::path(pathToAmbiant).stem().string(), "ambiant")) + if (boost::algorithm::icontains(fs::path(pathToAmbient).stem().string(), "ambient")) { - imageFloat = imageFloat - imageAmbiant; + imageFloat = imageFloat - imageAmbient; } intensityScaling(intList.at(i), imageFloat); @@ -400,7 +453,7 @@ void photometricStereo(const std::vector& imageList, currentPicture.block(2, 0, 1, currentMaskSize) * 0.0722; } - Eigen::MatrixXf M_channel(3, currentMaskSize); + Eigen::MatrixXf M_channel(lightMat.cols(), currentMaskSize); int currentIdx; if (PSParameters.isRobust) @@ -444,11 +497,11 @@ void photometricStereo(const std::vector& imageList, } } - for (size_t i = 0; i < maskSize; ++i) + for (size_t i = 0; i < currentMaskSize; ++i) { if (hasMask) { - currentIdx = indices.at(i); // Index in picture + currentIdx = currentMaskIndices.at(i); // Index in picture } else { @@ -461,17 +514,17 @@ void photometricStereo(const std::vector& imageList, for (size_t ch = 0; ch < 3; ++ch) { // Create I matrix for current pixel - Eigen::MatrixXf pixelValues_channel(imageList.size(), maskSize); + Eigen::MatrixXf pixelValues_channel(imageList.size(), currentMaskSize); for (size_t i = 0; i < imageList.size(); ++i) { - pixelValues_channel.block(i, 0, 1, maskSize) = imMat.block(ch + 3 * i, 0, 1, maskSize); + pixelValues_channel.block(i, 0, 1, currentMaskSize) = imMat.block(ch + 3 * i, 0, 1, currentMaskSize); } - for (size_t i = 0; i < maskSize; ++i) + for (size_t i = 0; i < currentMaskSize; ++i) { if (hasMask) { - currentIdx = indices.at(i); // Index in picture + currentIdx = currentMaskIndices.at(i); // Index in picture } else { @@ -506,10 +559,10 @@ void photometricStereo(const std::vector& imageList, for (size_t ch = 0; ch < 3; ++ch) { // Create I matrix for current pixel - Eigen::MatrixXf pixelValues_channel(imageList.size(), maskSize); + Eigen::MatrixXf pixelValues_channel(imageList.size(), currentMaskSize); for (size_t i = 0; i < imageList.size(); ++i) { - pixelValues_channel.block(i, 0, 1, maskSize) = imMat.block(ch + 3 * i, 0, 1, maskSize); + pixelValues_channel.block(i, 0, 1, currentMaskSize) = imMat.block(ch + 3 * i, 0, 1, currentMaskSize); } M_channel = lightMat.bdcSvd(Eigen::ComputeThinU | Eigen::ComputeThinV).solve(pixelValues_channel); @@ -541,7 +594,7 @@ void photometricStereo(const std::vector& imageList, albedo = albedoIm; } -void loadPSData(const std::string& folderPath, const size_t HS_order, std::vector>& intList, Eigen::MatrixXf& lightMat) +void loadPSData(const std::string& folderPath, const size_t SH_order, std::vector>& intList, Eigen::MatrixXf& lightMat) { std::string intFileName; std::string pathToCM; @@ -560,15 +613,15 @@ void loadPSData(const std::string& folderPath, const size_t HS_order, std::vecto } // Light directions - if (HS_order == 0) + if (SH_order == 0) { dirFileName = folderPath + "/light_directions.txt"; loadLightDirections(dirFileName, convertionMatrix, lightMat); } - else if (HS_order == 2) + else if (SH_order == 2) { - dirFileName = folderPath + "/light_directions_HS.txt"; - loadLightHS(dirFileName, lightMat); + dirFileName = folderPath + "/light_directions_SH.txt"; + loadLightSH(dirFileName, lightMat); } } @@ -585,7 +638,7 @@ void getPicturesNames(const std::string& folderPath, std::vector& i std::transform(fileExtension.begin(), fileExtension.end(), fileExtension.begin(), ::tolower); if (!boost::algorithm::icontains(currentFilePath.stem().string(), "mask") && - !boost::algorithm::icontains(currentFilePath.stem().string(), "ambiant")) + !boost::algorithm::icontains(currentFilePath.stem().string(), "ambient")) { for (const std::string& extension : extensions) { diff --git a/src/aliceVision/photometricStereo/photometricStereo.hpp b/src/aliceVision/photometricStereo/photometricStereo.hpp index 4348addeb3..3b72721cc1 100644 --- a/src/aliceVision/photometricStereo/photometricStereo.hpp +++ b/src/aliceVision/photometricStereo/photometricStereo.hpp @@ -19,7 +19,7 @@ namespace photometricStereo { struct PhotometricSteroParameters { size_t SHOrder; // Order of spherical harmonics (lighting) - bool removeAmbiant; // Do we remove ambiant light ? (currently tested) + bool removeAmbient; // Do we remove ambient light ? (currently tested) bool isRobust; // Do we use the robust version of the algorithm ? (currently tested) int downscale; // Downscale factor }; @@ -65,7 +65,7 @@ void photometricStereo(const sfmData::SfMData& sfmData, * @param[in] intList List of light intensities * @param[in] lightMat List of light direction/coefficients (SH) * @param[in] mask Mask that defines region of interest - * @param[in] pathToAmbiant Path to picture without any additional lighting + * @param[in] pathToAmbient Path to picture without any additional lighting * @param[in] PSParameters Parameters for the PS algorithm * @param[out] normals Normal map of the scene * @param[out] albedo Albedo map of the scene @@ -74,7 +74,7 @@ void photometricStereo(const std::vector& imageList, const std::vector>& intList, const Eigen::MatrixXf& lightMat, image::Image& mask, - const std::string& pathToAmbiant, + const std::string& pathToAmbient, const PhotometricSteroParameters& PSParameters, image::Image& normals, image::Image& albedo); @@ -82,11 +82,11 @@ void photometricStereo(const std::vector& imageList, /** * @brief Load data used in the PS algorithm * @param[in] folderPath Path to the folder that contains data - * @param[in] HS_order Order of the spherical harmonics + * @param[in] SH_order Order of the spherical harmonics * @param[out] intList Intensities of lights * @param[out] lightMat Directions of lights */ -void loadPSData(const std::string& folderPath, const size_t HS_order, std::vector>& intList, Eigen::MatrixXf& lightMat); +void loadPSData(const std::string& folderPath, const size_t SH_order, std::vector>& intList, Eigen::MatrixXf& lightMat); /** * @brief Get the name of the pictures in a given folder diff --git a/src/aliceVision/sphereDetection/sphereDetection.cpp b/src/aliceVision/sphereDetection/sphereDetection.cpp index 05b95ea31b..48ceb3da60 100644 --- a/src/aliceVision/sphereDetection/sphereDetection.cpp +++ b/src/aliceVision/sphereDetection/sphereDetection.cpp @@ -183,7 +183,7 @@ void sphereDetection(const sfmData::SfMData& sfmData, Ort::Session& session, fs: const std::string sphereName = std::to_string(viewID.second->getViewId()); const fs::path imagePath = fs::path(sfmData.getView(viewID.second->getViewId()).getImage().getImagePath()); - if (boost::algorithm::icontains(imagePath.stem().string(), "ambiant")) + if (boost::algorithm::icontains(imagePath.stem().string(), "ambient")) continue; const auto pred = predict(session, imagePath, minScore); @@ -219,7 +219,7 @@ void sphereDetection(const sfmData::SfMData& sfmData, Ort::Session& session, fs: ALICEVISION_LOG_WARNING("No sphere detected for '" << imagePath << "'."); } } - bpt::write_json(outputPath.append("detection.json").string(), fileTree); + bpt::write_json(outputPath.string(), fileTree); } void writeManualSphereJSON(const sfmData::SfMData& sfmData, const std::array& sphereParam, fs::path outputPath) @@ -246,7 +246,7 @@ void writeManualSphereJSON(const sfmData::SfMData& sfmData, const std::array hash; IndexT tmpPoseID = hash(parentPath.string()); // use a temporary pose Id to group the images @@ -802,24 +805,24 @@ int aliceVision_main(int argc, char** argv) { for (const auto& poseGroup : poseGroups) { - bool hasAmbiant = false; + bool hasAmbient = false; - // Photometric stereo : ambiant viewId used for all pictures + // Photometric stereo : ambient viewId used for all pictures for (const IndexT vId : poseGroup.second) { const fs::path imagePath = fs::path(sfmData.getView(vId).getImage().getImagePath()); - if (boost::algorithm::icontains(imagePath.stem().string(), "ambiant")) + if (boost::algorithm::icontains(imagePath.stem().string(), "ambient")) { - hasAmbiant = true; + hasAmbient = true; for (const auto it : poseGroup.second) { - // Update poseId with ambiant view id + // Update poseId with ambient view id sfmData.getView(it).setPoseId(vId); } break; } } - if (!hasAmbiant) + if (!hasAmbient) { // Sort views of the poseGroup per timestamps std::vector> sortedViews; diff --git a/src/software/pipeline/main_depthMapEstimation.cpp b/src/software/pipeline/main_depthMapEstimation.cpp index 4d46a50781..fb47c71243 100644 --- a/src/software/pipeline/main_depthMapEstimation.cpp +++ b/src/software/pipeline/main_depthMapEstimation.cpp @@ -296,7 +296,7 @@ int aliceVision_main(int argc, char* argv[]) } // MultiViewParams initialization - mvsUtils::MultiViewParams mp(sfmData, imagesFolder, outputFolder, "", false, downscale); + mvsUtils::MultiViewParams mp(sfmData, imagesFolder, outputFolder, "", mvsUtils::EFileType::none, downscale); // set MultiViewParams min/max view angle mp.setMinViewAngle(minViewAngle); diff --git a/src/software/pipeline/main_depthMapFiltering.cpp b/src/software/pipeline/main_depthMapFiltering.cpp index 56b61c3ad9..17c98ce92e 100644 --- a/src/software/pipeline/main_depthMapFiltering.cpp +++ b/src/software/pipeline/main_depthMapFiltering.cpp @@ -107,7 +107,7 @@ int aliceVision_main(int argc, char* argv[]) } // initialization - mvsUtils::MultiViewParams mp(sfmData, "", depthMapsFolder, outputFolder, "", true); + mvsUtils::MultiViewParams mp(sfmData, "", depthMapsFolder, outputFolder, mvsUtils::EFileType::depthMap); mp.setMinViewAngle(minViewAngle); mp.setMaxViewAngle(maxViewAngle); diff --git a/src/software/pipeline/main_lidarMeshing.cpp b/src/software/pipeline/main_lidarMeshing.cpp index 1d717ec021..bb2d94a25f 100644 --- a/src/software/pipeline/main_lidarMeshing.cpp +++ b/src/software/pipeline/main_lidarMeshing.cpp @@ -49,7 +49,7 @@ bool computeSubMesh(const std::string& pathSfmData, std::string& outputFile, con ALICEVISION_LOG_INFO("Loading source done"); // Create multiview params - mvsUtils::MultiViewParams mp(sfmData, "", "", "", false); + mvsUtils::MultiViewParams mp(sfmData); mp.userParams.put("LargeScale.forcePixelSize", 0.01); mp.userParams.put("LargeScale.forceWeight", 32.0); mp.userParams.put("LargeScale.helperPointsGridSize", 10); diff --git a/src/software/pipeline/main_lightingCalibration.cpp b/src/software/pipeline/main_lightingCalibration.cpp index 3c4a5c1a76..0a5d091fd0 100644 --- a/src/software/pipeline/main_lightingCalibration.cpp +++ b/src/software/pipeline/main_lightingCalibration.cpp @@ -13,6 +13,7 @@ // Lighting calibration #include +#include // Command line parameters #include @@ -47,18 +48,20 @@ int aliceVision_main(int argc, char** argv) system::Timer timer; std::string inputPath; - std::string inputJSON; + std::string inputDetection; std::string ouputJSON; std::string method; + bool doDebug; bool saveAsModel; + bool ellipticEstimation; // clang-format off po::options_description requiredParams("Required parameters"); requiredParams.add_options() ("inputPath,i", po::value(&inputPath)->required(), "Path to the SfMData input.") - ("inputJSON, j", po::value(&inputJSON)->required(), - "Path to the folder containing the JSON file that describes spheres' positions and radius.") + ("inputDetection, j", po::value(&inputDetection)->required(), + "Path to the folder containing the JSON file that describes spheres' positions and radius") ("outputFile, o", po::value(&ouputJSON)->required(), "Path to JSON output file."); @@ -67,7 +70,11 @@ int aliceVision_main(int argc, char** argv) ("saveAsModel, s", po::value(&saveAsModel)->default_value(false), "Calibration used for several datasets.") ("method, m", po::value(&method)->default_value("brightestPoint"), - "Method for light estimation."); + "Method for light estimation.") + ("doDebug, d", po::value(&doDebug)->default_value(true), + "Do we save debug images.") + ("ellipticEstimation, e", po::value(&ellipticEstimation)->default_value(false), + "Used ellipse model for calibration spheres (more precise)."); // clang-format on CmdLine cmdline("AliceVision lightingCalibration"); @@ -84,6 +91,11 @@ int aliceVision_main(int argc, char** argv) ALICEVISION_LOG_ERROR("Directory input: WIP"); ALICEVISION_THROW(std::invalid_argument, "Input directories are not yet supported"); } + else if (fs::path(inputDetection).extension() != ".json") + { + ALICEVISION_LOG_ERROR("The input detection file must be a JSON file."); + ALICEVISION_THROW(std::invalid_argument, "JSON needed for sphere positions and radius."); + } else { sfmData::SfMData sfmData; @@ -92,7 +104,8 @@ int aliceVision_main(int argc, char** argv) ALICEVISION_LOG_ERROR("The input file '" + inputPath + "' cannot be read."); return EXIT_FAILURE; } - lightingEstimation::lightCalibration(sfmData, inputJSON, ouputJSON, method, saveAsModel); + + lightingEstimation::lightCalibration(sfmData, inputDetection, ouputJSON, method, doDebug, saveAsModel, ellipticEstimation); } ALICEVISION_LOG_INFO("Task done in (s): " + std::to_string(timer.elapsed())); diff --git a/src/software/pipeline/main_meshing.cpp b/src/software/pipeline/main_meshing.cpp index 19ffa18a62..8c8ffea046 100644 --- a/src/software/pipeline/main_meshing.cpp +++ b/src/software/pipeline/main_meshing.cpp @@ -306,7 +306,7 @@ int aliceVision_main(int argc, char* argv[]) } // initialization - mvsUtils::MultiViewParams mp(sfmData, "", "", depthMapsFolder, meshingFromDepthMaps); + mvsUtils::MultiViewParams mp(sfmData, "", "", depthMapsFolder, meshingFromDepthMaps ? mvsUtils::EFileType::depthMap : mvsUtils::EFileType::none); mp.userParams.put("LargeScale.universePercentile", universePercentile); mp.userParams.put("LargeScale.helperPointsGridSize", helperPointsGridSize); diff --git a/src/software/pipeline/main_photometricStereo.cpp b/src/software/pipeline/main_photometricStereo.cpp index 0fd4da18b4..4b605bb840 100644 --- a/src/software/pipeline/main_photometricStereo.cpp +++ b/src/software/pipeline/main_photometricStereo.cpp @@ -78,9 +78,9 @@ int aliceVision_main(int argc, char** argv) ("pathToJSONLightFile,l", po::value(&pathToLightData)->default_value("defaultJSON.txt"), "Path to light file (JSON). If empty, .txt files are expected in the image folder.") ("SHOrder,s", po::value(&PSParameters.SHOrder)->default_value(0), - "Spherical harmonics order, 0 = directional, 1 = directional + ambiant, 2 = second order SH.") - ("removeAmbiant,a", po::value(&PSParameters.removeAmbiant)->default_value(false), - "True if the ambiant light is to be removed on PS images, false otherwise.") + "Spherical harmonics order, 0 = directional, 1 = directional + ambient, 2 = second order SH.") + ("removeAmbient,a", po::value(&PSParameters.removeAmbient)->default_value(false), + "True if the ambient light is to be removed on PS images, false otherwise.") ("isRobust,r", po::value(&PSParameters.isRobust)->default_value(false), "True to use the robust algorithm, false otherwise.") ("downscale, d", po::value(&PSParameters.downscale)->default_value(1), @@ -118,7 +118,6 @@ int aliceVision_main(int argc, char** argv) ALICEVISION_LOG_ERROR("The input file '" + inputPath + "' cannot be read."); return EXIT_FAILURE; } - photometricStereo::photometricStereo(sfmData, pathToLightData, maskPath, outputPath, PSParameters, normalsIm, albedoIm); } diff --git a/src/software/pipeline/main_texturing.cpp b/src/software/pipeline/main_texturing.cpp index 0be28738fe..9ac82a2673 100644 --- a/src/software/pipeline/main_texturing.cpp +++ b/src/software/pipeline/main_texturing.cpp @@ -50,6 +50,8 @@ int aliceVision_main(int argc, char* argv[]) std::string outputFolder; std::string imagesFolder; + std::string normalsFolder; + image::EImageColorSpace workingColorSpace = image::EImageColorSpace::SRGB; image::EImageColorSpace outputColorSpace = image::EImageColorSpace::AUTO; bool flipNormals = false; @@ -81,6 +83,9 @@ int aliceVision_main(int argc, char* argv[]) ("imagesFolder", po::value(&imagesFolder), "Use images from a specific folder instead of those specify in the SfMData file.\n" "Filename should be the image UID.") + ("normalsFolder", po::value(&normalsFolder), + "Use normal maps from a specific folder to texture the mesh.\n" + "Filename should be: UID_normalMap.") ("textureSide", po::value(&texParams.textureSide)->default_value(texParams.textureSide), "Output texture size.") ("downscale", po::value(&texParams.downscale)->default_value(texParams.downscale), @@ -248,6 +253,14 @@ int aliceVision_main(int argc, char* argv[]) mesh.generateNormalAndHeightMaps(mp, denseMesh, outputFolder, bumpMappingParams); } + // generate normal maps textures from the fusion of normal maps per image + if (!normalsFolder.empty() && bumpMappingParams.bumpMappingFileType != image::EImageFileType::NONE) + { + ALICEVISION_LOG_INFO("Generate normal maps."); + mvsUtils::MultiViewParams mpN(sfmData, normalsFolder); + mesh.generateTextures(mpN, outputFolder, hwc.getMaxMemory(), texParams.textureFileType, mvsUtils::EFileType::normalMap); + } + // save final obj file if (!inputMeshFilepath.empty()) { diff --git a/src/software/utils/main_lightingEstimation.cpp b/src/software/utils/main_lightingEstimation.cpp index ac2ea50fbc..6e11c0b4f3 100644 --- a/src/software/utils/main_lightingEstimation.cpp +++ b/src/software/utils/main_lightingEstimation.cpp @@ -331,7 +331,7 @@ int main(int argc, char** argv) } // initialization - mvsUtils::MultiViewParams mp(sfmData, imagesFolder, "", depthMapsFilterFolder, false); + mvsUtils::MultiViewParams mp(sfmData, imagesFolder, "", depthMapsFilterFolder, mvsUtils::EFileType::none); lightingEstimation::LighthingEstimator estimator;